Created
June 1, 2015 16:01
-
-
Save lexinator/f5103cee09b514b2778e to your computer and use it in GitHub Desktop.
aurora exceeding default timeout to replicated log
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| org.apache.aurora.scheduler.storage.Storage$StorageException: There was a problem committing the transaction to the log. | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:652) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:642) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage$3.apply(MemStorage.java:147) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage$3.apply(MemStorage.java:144) | |
| at org.apache.aurora.scheduler.storage.db.DbStorage.write(DbStorage.java:137) | |
| at org.mybatis.guice.transactional.TransactionalMethodInterceptor.invoke(TransactionalMethodInterceptor.java:101) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:144) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage.doInTransaction(LogStorage.java:642) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage.write(LogStorage.java:675) | |
| at org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.write(CallOrderEnforcingStorage.java:122) | |
| at org.apache.aurora.scheduler.UserTaskLauncher.statusUpdate(UserTaskLauncher.java:86) | |
| at org.apache.aurora.scheduler.mesos.MesosSchedulerImpl.statusUpdate(MesosSchedulerImpl.java:230) | |
| at org.apache.aurora.GuiceUtils$3.invoke(GuiceUtils.java:120) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| Caused by: org.apache.aurora.scheduler.log.Log$Stream$StreamAccessException: Timeout performing log append | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.disableLog(MesosLog.java:352) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.mutate(MesosLog.java:367) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.append(MesosLog.java:315) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.append(MesosLog.java:145) | |
| at org.apache.aurora.scheduler.storage.log.StreamManagerImpl.appendAndGetPosition(StreamManagerImpl.java:239) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| at org.apache.aurora.scheduler.storage.log.StreamManagerImpl$StreamTransactionImpl.commit(StreamManagerImpl.java:268) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:647) | |
| ... 14 more | |
| Caused by: java.util.concurrent.TimeoutException: Timed out while attempting to append | |
| at org.apache.mesos.Log$Writer.append(Native Method) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLogStreamModule$5.append(MesosLogStreamModule.java:188) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream$3.apply(MesosLog.java:319) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream$3.apply(MesosLog.java:315) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.mutate(MesosLog.java:365) | |
| ... 20 more | |
| E0531 16:41:55.387 THREAD1446 org.apache.aurora.scheduler.mesos.MesosSchedulerImpl.statusUpdate: Status update failed due to scheduler exception: org.apache.aurora.scheduler.storage.Storage$StorageException: There was a problem committing the transaction to the log. | |
| org.apache.aurora.scheduler.storage.Storage$StorageException: There was a problem committing the transaction to the log. | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:652) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:642) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage$3.apply(MemStorage.java:147) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage$3.apply(MemStorage.java:144) | |
| at org.apache.aurora.scheduler.storage.db.DbStorage.write(DbStorage.java:137) | |
| at org.mybatis.guice.transactional.TransactionalMethodInterceptor.invoke(TransactionalMethodInterceptor.java:101) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:144) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage.doInTransaction(LogStorage.java:642) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage.write(LogStorage.java:675) | |
| at org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.write(CallOrderEnforcingStorage.java:122) | |
| at org.apache.aurora.scheduler.UserTaskLauncher.statusUpdate(UserTaskLauncher.java:86) | |
| at org.apache.aurora.scheduler.mesos.MesosSchedulerImpl.statusUpdate(MesosSchedulerImpl.java:230) | |
| at org.apache.aurora.GuiceUtils$3.invoke(GuiceUtils.java:120) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| Caused by: org.apache.aurora.scheduler.log.Log$Stream$StreamAccessException: Timeout performing log append | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.disableLog(MesosLog.java:352) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.mutate(MesosLog.java:367) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.append(MesosLog.java:315) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.append(MesosLog.java:145) | |
| at org.apache.aurora.scheduler.storage.log.StreamManagerImpl.appendAndGetPosition(StreamManagerImpl.java:239) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| at org.apache.aurora.scheduler.storage.log.StreamManagerImpl$StreamTransactionImpl.commit(StreamManagerImpl.java:268) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:647) | |
| ... 14 more | |
| Caused by: java.util.concurrent.TimeoutException: Timed out while attempting to append | |
| at org.apache.mesos.Log$Writer.append(Native Method) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLogStreamModule$5.append(MesosLogStreamModule.java:188) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream$3.apply(MesosLog.java:319) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream$3.apply(MesosLog.java:315) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.mutate(MesosLog.java:365) | |
| ... 20 more | |
| Exception in thread "Thread-1305" org.apache.aurora.scheduler.storage.Storage$StorageException: There was a problem committing the transaction to the log. | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:652) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:642) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage$3.apply(MemStorage.java:147) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage$3.apply(MemStorage.java:144) | |
| at org.apache.aurora.scheduler.storage.db.DbStorage.write(DbStorage.java:137) | |
| at org.mybatis.guice.transactional.TransactionalMethodInterceptor.invoke(TransactionalMethodInterceptor.java:101) | |
| at org.apache.aurora.scheduler.storage.mem.MemStorage.write(MemStorage.java:144) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage.doInTransaction(LogStorage.java:642) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage.write(LogStorage.java:675) | |
| at org.apache.aurora.scheduler.storage.CallOrderEnforcingStorage.write(CallOrderEnforcingStorage.java:122) | |
| at org.apache.aurora.scheduler.UserTaskLauncher.statusUpdate(UserTaskLauncher.java:86) | |
| at org.apache.aurora.scheduler.mesos.MesosSchedulerImpl.statusUpdate(MesosSchedulerImpl.java:230) | |
| at org.apache.aurora.GuiceUtils$3.invoke(GuiceUtils.java:120) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| Caused by: org.apache.aurora.scheduler.log.Log$Stream$StreamAccessException: Timeout performing log append | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.disableLog(MesosLog.java:352) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.mutate(MesosLog.java:367) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.append(MesosLog.java:315) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.append(MesosLog.java:145) | |
| at org.apache.aurora.scheduler.storage.log.StreamManagerImpl.appendAndGetPosition(StreamManagerImpl.java:239) | |
| at com.twitter.common.inject.TimedInterceptor.invoke(TimedInterceptor.java:87) | |
| at org.apache.aurora.scheduler.storage.log.StreamManagerImpl$StreamTransactionImpl.commit(StreamManagerImpl.java:268) | |
| at org.apache.aurora.scheduler.storage.log.LogStorage$24.apply(LogStorage.java:647) | |
| ... 14 more | |
| Caused by: java.util.concurrent.TimeoutException: Timed out while attempting to append | |
| at org.apache.mesos.Log$Writer.append(Native Method) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLogStreamModule$5.append(MesosLogStreamModule.java:188) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream$3.apply(MesosLog.java:319) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream$3.apply(MesosLog.java:315) | |
| at org.apache.aurora.scheduler.log.mesos.MesosLog$LogStream.mutate(MesosLog.java:365) | |
| ... 20 more | |
| I0531 16:41:55.390561 9769 sched.cpp:1320] Asked to abort the driver | |
| I0531 16:41:55.391 THREAD114 com.twitter.common.application.ShutdownRegistry$ShutdownRegistryImpl.execute: Action controller has already completed, subsequent calls ignored. | |
| Heap | |
| PSYoungGen total 633344K, used 23873K [0x00000007d5500000, 0x0000000800000000, 0x0000000800000000) | |
| eden space 570880K, 3% used [0x00000007d5500000,0x00000007d6898748,0x00000007f8280000) | |
| from space 62464K, 6% used [0x00000007f8280000,0x00000007f8638000,0x00000007fbf80000) | |
| to space 59392K, 0% used [0x00000007fc600000,0x00000007fc600000,0x0000000800000000) | |
| ParOldGen total 1398272K, used 359042K [0x000000077ff80000, 0x00000007d5500000, 0x00000007d5500000) | |
| object space 1398272K, 25% used [0x000000077ff80000,0x0000000795e20a88,0x00000007d5500000) | |
| PSPermGen total 46080K, used 45639K [0x000000077ad80000, 0x000000077da80000, 0x000000077ff80000) | |
| object space 46080K, 99% used [0x000000077ad80000,0x000000077da11e98,0x000000077da80000) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment