Created
July 5, 2021 18:22
-
-
Save nsivabalan/c944581536e8c79defb7e31678f1aabf to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| 21/07/05 15:59:07 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=17518.0%2F273309f4-674f-41fe-80c9-bb0693d38b77-375_196-12-9655_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:07 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:07 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:07 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 185647.0 787af4e9-70fd-4f1b-aaad-5ea4e1f305e8-374_224-12-9683_20210705142539.parquet | |
| 21/07/05 15:59:07 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=185647.0%2F787af4e9-70fd-4f1b-aaad-5ea4e1f305e8-374_224-12-9683_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:07 INFO io.HoodieCreateHandle: CreateHandle for partitionPath 180306.0 fileID 103d43cf-9452-46c4-ab9d-cccbaa2212b1-373, took 2165 ms. | |
| 21/07/05 15:59:07 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:07 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:07 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 169858.0 857921f2-f84d-407c-b776-b293e2c1e0c2-376_182-12-9641_20210705142539.parquet | |
| 21/07/05 15:59:07 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=169858.0%2F857921f2-f84d-407c-b776-b293e2c1e0c2-376_182-12-9641_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:07 INFO io.HoodieCreateHandle: CreateHandle for partitionPath 154062.0 fileID 229c1efc-3a19-4058-af2e-ef9ef4f009cf-373, took 2337 ms. | |
| 21/07/05 15:59:07 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:07 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:07 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 148546.0 e5392751-389a-4c37-b033-57fac808fe23-374_126-12-9585_20210705142539.parquet | |
| 21/07/05 15:59:07 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=148546.0%2Fe5392751-389a-4c37-b033-57fac808fe23-374_126-12-9585_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Putting block rdd_18_224 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Putting block rdd_18_196 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Block rdd_18_196 could not be removed as it was not found on disk or in memory | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Block rdd_18_224 could not be removed as it was not found on disk or in memory | |
| 21/07/05 15:59:37 ERROR executor.Executor: Exception in task 99.0 in stage 12.0 (TID 9683) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 ERROR executor.Executor: Exception in task 71.0 in stage 12.0 (TID 9655) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9724 | |
| 21/07/05 15:59:37 INFO executor.Executor: Running task 19.1 in stage 12.0 (TID 9724) | |
| 21/07/05 15:59:37 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9725 | |
| 21/07/05 15:59:37 INFO executor.Executor: Running task 48.1 in stage 12.0 (TID 9725) | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (106.4 MiB) non-empty blocks including 294 (7.4 MiB) local and 0 (0.0 B) host-local and 3935 (99.0 MiB) remote blocks | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (122.3 MiB) non-empty blocks including 294 (8.5 MiB) local and 0 (0.0 B) host-local and 3935 (113.8 MiB) remote blocks | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Started 5 remote fetches in 7 ms | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Started 6 remote fetches in 9 ms | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Putting block rdd_18_182 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Block rdd_18_182 could not be removed as it was not found on disk or in memory | |
| 21/07/05 15:59:37 ERROR executor.Executor: Exception in task 57.0 in stage 12.0 (TID 9641) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9743 | |
| 21/07/05 15:59:37 INFO executor.Executor: Running task 56.1 in stage 12.0 (TID 9743) | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (101.8 MiB) non-empty blocks including 294 (7.0 MiB) local and 0 (0.0 B) host-local and 3935 (94.8 MiB) remote blocks | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Started 6 remote fetches in 27 ms | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Putting block rdd_18_126 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 15:59:37 WARN storage.BlockManager: Block rdd_18_126 could not be removed as it was not found on disk or in memory | |
| 21/07/05 15:59:37 ERROR executor.Executor: Exception in task 1.0 in stage 12.0 (TID 9585) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 15:59:37 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9768 | |
| 21/07/05 15:59:37 INFO executor.Executor: Running task 40.1 in stage 12.0 (TID 9768) | |
| 21/07/05 15:59:37 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (121.5 MiB) non-empty blocks including 294 (8.5 MiB) local and 0 (0.0 B) host-local and 3935 (113.1 MiB) remote blocks | |
| 21/07/05 15:59:38 INFO storage.ShuffleBlockFetcherIterator: Started 5 remote fetches in 20 ms | |
| 21/07/05 15:59:49 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 15:59:49 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 15:59:49 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 15:59:49 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 15:59:49 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:49 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:49 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 15542.0 708eee2b-3440-4a5a-8e44-bc097a7020e7-0_144-12-9724_20210705142539.parquet | |
| 21/07/05 15:59:49 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=15542.0%2F708eee2b-3440-4a5a-8e44-bc097a7020e7-0_144-12-9724_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:49 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:49 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:49 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 169172.0 98376d80-95fe-412b-8713-4be302e38655-0_181-12-9743_20210705142539.parquet | |
| 21/07/05 15:59:49 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=169172.0%2F98376d80-95fe-412b-8713-4be302e38655-0_181-12-9743_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:50 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 15:59:50 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 15:59:50 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:50 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:50 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 166127.0 2948781d-961e-4422-b4e4-5e84dedf9e3e-0_173-12-9725_20210705142539.parquet | |
| 21/07/05 15:59:50 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=166127.0%2F2948781d-961e-4422-b4e4-5e84dedf9e3e-0_173-12-9725_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 15:59:50 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 15:59:50 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 15:59:50 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 15:59:50 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 15:59:50 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 163030.0 21506d4b-0497-4f72-99d3-b448c4e5d6ba-0_165-12-9768_20210705142539.parquet | |
| 21/07/05 15:59:50 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=163030.0%2F21506d4b-0497-4f72-99d3-b448c4e5d6ba-0_165-12-9768_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 16:00:19 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:19 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:19 WARN storage.BlockManager: Putting block rdd_18_144 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:00:19 WARN storage.BlockManager: Block rdd_18_144 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:00:19 ERROR executor.Executor: Exception in task 19.1 in stage 12.0 (TID 9724) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:19 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9805 | |
| 21/07/05 16:00:19 INFO executor.Executor: Running task 57.2 in stage 12.0 (TID 9805) | |
| 21/07/05 16:00:19 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (116.1 MiB) non-empty blocks including 294 (8.1 MiB) local and 0 (0.0 B) host-local and 3935 (108.0 MiB) remote blocks | |
| 21/07/05 16:00:19 INFO storage.ShuffleBlockFetcherIterator: Started 5 remote fetches in 7 ms | |
| 21/07/05 16:00:20 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:20 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:20 WARN storage.BlockManager: Putting block rdd_18_181 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:00:20 WARN storage.BlockManager: Block rdd_18_181 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:00:20 ERROR executor.Executor: Exception in task 56.1 in stage 12.0 (TID 9743) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:20 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9807 | |
| 21/07/05 16:00:20 INFO executor.Executor: Running task 7.2 in stage 12.0 (TID 9807) | |
| 21/07/05 16:00:20 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (101.1 MiB) non-empty blocks including 294 (7.0 MiB) local and 0 (0.0 B) host-local and 3935 (94.1 MiB) remote blocks | |
| 21/07/05 16:00:20 INFO storage.ShuffleBlockFetcherIterator: Started 6 remote fetches in 4 ms | |
| 21/07/05 16:00:20 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:20 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:20 WARN storage.BlockManager: Putting block rdd_18_173 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:00:20 WARN storage.BlockManager: Block rdd_18_173 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:00:20 ERROR executor.Executor: Exception in task 48.1 in stage 12.0 (TID 9725) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:20 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9810 | |
| 21/07/05 16:00:20 INFO executor.Executor: Running task 23.2 in stage 12.0 (TID 9810) | |
| 21/07/05 16:00:20 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (119.1 MiB) non-empty blocks including 294 (8.3 MiB) local and 0 (0.0 B) host-local and 3935 (110.7 MiB) remote blocks | |
| 21/07/05 16:00:20 INFO storage.ShuffleBlockFetcherIterator: Started 5 remote fetches in 5 ms | |
| 21/07/05 16:00:21 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:21 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:21 WARN storage.BlockManager: Putting block rdd_18_165 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:00:21 WARN storage.BlockManager: Block rdd_18_165 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:00:21 ERROR executor.Executor: Exception in task 40.1 in stage 12.0 (TID 9768) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more | |
| Caused by: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:143) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:77) | |
| ... 24 more | |
| Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at java.util.concurrent.FutureTask.report(FutureTask.java:122) | |
| at java.util.concurrent.FutureTask.get(FutureTask.java:192) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.execute(BoundedInMemoryExecutor.java:141) | |
| ... 25 more | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| ... 3 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:00:21 INFO executor.YarnCoarseGrainedExecutorBackend: Got assigned task 9821 | |
| 21/07/05 16:00:21 INFO executor.Executor: Running task 65.2 in stage 12.0 (TID 9821) | |
| 21/07/05 16:00:21 INFO storage.ShuffleBlockFetcherIterator: Getting 4229 (118.2 MiB) non-empty blocks including 294 (8.2 MiB) local and 0 (0.0 B) host-local and 3935 (110.0 MiB) remote blocks | |
| 21/07/05 16:00:21 INFO storage.ShuffleBlockFetcherIterator: Started 5 remote fetches in 6 ms | |
| 21/07/05 16:00:25 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 16:00:25 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 16:00:25 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 16:00:25 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 16:00:25 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 150548.0 d69bf12a-2a47-4887-915e-cf93b5527332-0_132-12-9807_20210705142539.parquet | |
| 21/07/05 16:00:25 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=150548.0%2Fd69bf12a-2a47-4887-915e-cf93b5527332-0_132-12-9807_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 16:00:25 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 16:00:25 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 16:00:25 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 16:00:25 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 16:00:25 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 169519.0 857921f2-f84d-407c-b776-b293e2c1e0c2-0_182-12-9805_20210705142539.parquet | |
| 21/07/05 16:00:25 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=169519.0%2F857921f2-f84d-407c-b776-b293e2c1e0c2-0_182-12-9805_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 16:00:25 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 16:00:25 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 16:00:25 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 16:00:25 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 16:00:25 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 156856.0 76021b4b-0701-46c6-8029-43562657bdcf-0_148-12-9810_20210705142539.parquet | |
| 21/07/05 16:00:25 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=156856.0%2F76021b4b-0701-46c6-8029-43562657bdcf-0_148-12-9810_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 16:00:26 INFO queue.IteratorBasedQueueProducer: starting to buffer records | |
| 21/07/05 16:00:26 INFO queue.BoundedInMemoryExecutor: starting consumer thread | |
| 21/07/05 16:00:26 INFO marker.MarkerFilesFactory: Instantiated MarkerFiles with mode: TIMELINE_BASED | |
| 21/07/05 16:00:26 INFO view.FileSystemViewManager: Creating remote view for basePath s3a://siva-test-bucket-june-16/hudi_testing/hudi_base_2. Server=ip-172-31-45-49.us-east-2.compute.internal:37481, Timeout=300 | |
| 21/07/05 16:00:26 INFO marker.TimelineBasedMarkerFiles: ^^^ [timeline-based] Create marker file : 172519.0 ee8c1279-c380-4b51-ae24-419ced76b038-0_190-12-9821_20210705142539.parquet | |
| 21/07/05 16:00:26 INFO view.RemoteHoodieTableFileSystemView: Sending request : (http://ip-172-31-45-49.us-east-2.compute.internal:37481/v1/hoodie/view/marker/create?markername=172519.0%2Fee8c1279-c380-4b51-ae24-419ced76b038-0_190-12-9821_20210705142539.parquet.marker.CREATE&markerdirpath=s3a%3A%2F%2Fsiva-test-bucket-june-16%2Fhudi_testing%2Fhudi_base_2%2F.hoodie%2F.temp%2F20210705142539&timelinehash=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855) | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error consuming records | |
| org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Putting block rdd_18_148 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Putting block rdd_18_190 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Putting block rdd_18_182 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Block rdd_18_182 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Block rdd_18_190 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Block rdd_18_148 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Putting block rdd_18_132 failed due to exception java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error. | |
| 21/07/05 16:02:06 ERROR queue.BoundedInMemoryExecutor: error producing records | |
| org.apache.hudi.exception.HoodieException: operation has failed | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.throwExceptionIfFailed(BoundedInMemoryQueue.java:247) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueue.insertRecord(BoundedInMemoryQueue.java:184) | |
| at org.apache.hudi.common.util.queue.IteratorBasedQueueProducer.produce(IteratorBasedQueueProducer.java:46) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$0(BoundedInMemoryExecutor.java:92) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) | |
| at java.util.concurrent.FutureTask.run(FutureTask.java:266) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:485) | |
| at org.apache.hudi.table.marker.TimelineBasedMarkerFiles.create(TimelineBasedMarkerFiles.java:80) | |
| at org.apache.hudi.table.marker.MarkerFiles.create(MarkerFiles.java:54) | |
| at org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:181) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:83) | |
| at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:65) | |
| at org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:34) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:83) | |
| at org.apache.hudi.execution.CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteInsertHandler.java:40) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37) | |
| at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$2(BoundedInMemoryExecutor.java:121) | |
| ... 4 more | |
| Caused by: org.apache.http.client.HttpResponseException: Server Error | |
| at org.apache.http.impl.client.AbstractResponseHandler.handleResponse(AbstractResponseHandler.java:69) | |
| at org.apache.http.client.fluent.Response.handleResponse(Response.java:90) | |
| at org.apache.http.client.fluent.Response.returnContent(Response.java:97) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.executeRequest(RemoteHoodieTableFileSystemView.java:187) | |
| at org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView.createMarker(RemoteHoodieTableFileSystemView.java:483) | |
| ... 14 more | |
| 21/07/05 16:02:06 WARN storage.BlockManager: Block rdd_18_132 could not be removed as it was not found on disk or in memory | |
| 21/07/05 16:02:06 ERROR executor.Executor: Exception in task 57.2 in stage 12.0 (TID 9805) | |
| java.lang.RuntimeException: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121) | |
| at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:44) | |
| at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:484) | |
| at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:490) | |
| at org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:221) | |
| at org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:349) | |
| at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) | |
| at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) | |
| at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) | |
| at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) | |
| at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) | |
| at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) | |
| at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) | |
| at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) | |
| at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) | |
| at org.apache.spark.scheduler.Task.run(Task.scala:127) | |
| at org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:446) | |
| at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1377) | |
| at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:449) | |
| at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) | |
| at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) | |
| at java.lang.Thread.run(Thread.java:748) | |
| Caused by: org.apache.hudi.exception.HoodieException: org.apache.hudi.exception.HoodieException: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieRemoteException: Server Error | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:81) | |
| at org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:36) | |
| at org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119) | |
| ... 22 more |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment