Created
November 21, 2019 19:44
-
-
Save timcharper/d83e1f7c9e09fc758650c1c0ef6a4a1b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: [2019-11-19 18:44:23,612] INFO Migration for storage: Version(18, 200, 0, PERSISTENCE_STORE) to target: Version(19, 300, 0, PERSISTENCE_STORE): apply change for version: Version(19, 200, 0, PERSISTENCE_STORE) (mesosphere.marathon.storage.migration.Migration:scala-execution-context-global-65) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: [2019-11-19 18:44:23,646] INFO Starting migration to 1.9.200 (mesosphere.marathon.storage.migration.MigrationTo19200:scala-execution-context-global-64) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: [2019-11-19 18:44:23,698] INFO Leadership ended with failure; exiting (mesosphere.marathon.core.election.ElectionServiceImpl:marathon-akka.actor.default-dispatcher-4) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: mesosphere.marathon.MigrationFailedException: while migrating storage to major: 19 | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: minor: 200 | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: patch: 0 | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: format: PERSISTENCE_STORE | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at mesosphere.marathon.storage.migration.Migration$$anonfun$$nestedInanonfun$applyMigrationSteps$5$1.applyOrElse(Migration.scala:102) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at mesosphere.marathon.storage.migration.Migration$$anonfun$$nestedInanonfun$applyMigrationSteps$5$1.applyOrElse(Migration.scala:99) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:34) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.util.Failure.recover(Try.scala:230) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.concurrent.Future.$anonfun$recover$1(Future.scala:391) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.concurrent.Future$$Lambda$1273/1231491854.apply(Unknown Source) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:29) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:29) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.concurrent.impl.Promise$$Lambda$445/1703696921.apply(Unknown Source) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1402) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1689) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: Caused by: play.api.libs.json.JsResultException: JsResultException(errors:List((/agentInfo,List(JsonValidationError(List(error.path.missing),WrappedArray()))))) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at play.api.libs.json.JsReadable.$anonfun$as$2(JsReadable.scala:25) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at play.api.libs.json.JsReadable$$Lambda$803/792195577.apply(Unknown Source) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at play.api.libs.json.JsError.fold(JsResult.scala:56) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at play.api.libs.json.JsReadable.as(JsReadable.scala:24) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at play.api.libs.json.JsReadable.as$(JsReadable.scala:23) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at play.api.libs.json.JsObject.as(JsValue.scala:124) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at mesosphere.marathon.storage.migration.MigrationTo19200.extractInstanceFromJson(MigrationTo19200.scala:62) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at mesosphere.marathon.storage.migration.MigrationTo19200.$anonfun$instanceMigrationFlow$2(MigrationTo19200.scala:69) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at mesosphere.marathon.storage.migration.MigrationTo19200$$Lambda$1577/670905571.apply(Unknown Source) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.Map$$anon$1.onPush(Ops.scala:54) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.GraphInterpreter.processPush(GraphInterpreter.scala:523) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.GraphInterpreter.execute(GraphInterpreter.scala:409) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.GraphInterpreterShell.runBatch(ActorGraphInterpreter.scala:606) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.GraphInterpreterShell$AsyncInput.execute(ActorGraphInterpreter.scala:485) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.GraphInterpreterShell.processEvent(ActorGraphInterpreter.scala:581) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.ActorGraphInterpreter.akka$stream$impl$fusing$ActorGraphInterpreter$$processEvent(ActorGraphInterpreter.scala:749) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.ActorGraphInterpreter$$anonfun$receive$1.applyOrElse(ActorGraphInterpreter.scala:764) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.actor.Actor.aroundReceive(Actor.scala:539) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.actor.Actor.aroundReceive$(Actor.scala:537) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.stream.impl.fusing.ActorGraphInterpreter.aroundReceive(ActorGraphInterpreter.scala:671) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.actor.ActorCell.receiveMessage(ActorCell.scala:612) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.actor.ActorCell.invoke(ActorCell.scala:581) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.Mailbox.run(Mailbox.scala:229) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.Mailbox.exec(Mailbox.scala:241) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) | |
| Nov 19 18:44:23 ip-....internal marathon.sh[10865]: at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) | |
| Further restarts of Marathon could lead to following error messages | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: mesosphere.marathon.MigrationFailedException: Migration Failed: Migration is already in progress; /migration-in-progress node already exists | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at mesosphere.marathon.storage.migration.Migration$$anonfun$migrateStorage$1.applyOrElse(Migration.scala:188) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at mesosphere.marathon.storage.migration.Migration$$anonfun$migrateStorage$1.applyOrElse(Migration.scala:184) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.runtime.AbstractPartialFunction.apply(AbstractPartialFunction.scala:34) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.util.Failure.recover(Try.scala:230) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.Future.$anonfun$recover$1(Future.scala:391) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.Future$$Lambda$1271/833881059.apply(Unknown Source) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.impl.Promise.liftedTree1$1(Promise.scala:29) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.impl.Promise.$anonfun$transform$1(Promise.scala:29) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.impl.Promise$$Lambda$445/1703696921.apply(Unknown Source) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ForkJoinTask$RunnableExecuteAction.exec(ForkJoinTask.java:1402) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:289) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1056) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1689) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:157) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: Caused by: mesosphere.marathon.StoreCommandFailedException: Migration is already in progress; /migration-in-progress node already exists | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at mesosphere.marathon.core.storage.store.impl.zk.ZkPersistenceStore$stateMachine$macro$1583$1.apply(ZkPersistenceStore.scala:342) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at mesosphere.marathon.core.storage.store.impl.zk.ZkPersistenceStore$stateMachine$macro$1583$1.apply(ZkPersistenceStore.scala:336) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:60) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at java.lang.Thread.run(Thread.java:745) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.zookeeper.KeeperException.create(KeeperException.java:122) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at mesosphere.marathon.core.storage.store.impl.zk.ZkFuture.processResult(ZkFuture.scala:30) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.curator.framework.imps.CreateBuilderImpl.sendBackgroundResponse(CreateBuilderImpl.java:857) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.curator.framework.imps.CreateBuilderImpl.access$1400(CreateBuilderImpl.java:49) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.curator.framework.imps.CreateBuilderImpl$6.processResult(CreateBuilderImpl.java:658) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.curator.framework.imps.CreateBuilderImpl$7.processResult(CreateBuilderImpl.java:670) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:614) | |
| Nov 19 18:44:46 ip-....internal marathon.sh[11192]: at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment