kube-controller-manager AzureDisks

gistfile1.txt

// Killing the Replication Controllers [mongo-1, mongo-2, mongo-3]
I0929 15:31:51.658862       1 replication_controller.go:633] Replication Controller has been deleted integration/mongo-1
I0929 15:31:51.664271       1 garbagecollector.go:751] none of object [v1/Pod, namespace: integration, name: mongo-1-yb4vd, uid: fb906480-8656-11e6-b6aa-000d3ab30bf6]'s owners exist any more, will garbage collect it
I0929 15:31:53.631843       1 replication_controller.go:633] Replication Controller has been deleted integration/mongo-2
I0929 15:31:53.635477       1 garbagecollector.go:751] none of object [v1/Pod, namespace: integration, name: mongo-2-254io, uid: 6ba3a611-8657-11e6-b6aa-000d3ab30bf6]'s owners exist any more, will garbage collect it
I0929 15:31:55.787855       1 replication_controller.go:633] Replication Controller has been deleted integration/mongo-3
I0929 15:31:55.792872       1 garbagecollector.go:751] none of object [v1/Pod, namespace: integration, name: mongo-3-t6i12, uid: 6ba884e8-8657-11e6-b6aa-000d3ab30bf6]'s owners exist any more, will garbage collect it
W0929 15:32:49.351380       1 reflector.go:330] pkg/controller/garbagecollector/garbagecollector.go:758: watch of <nil> ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [19927/19119]) [20926]
E0929 15:35:25.118537       1 routecontroller.go:197] Error updating node preprod-k8s-cluster-node-8, retrying: Operation cannot be fulfilled on nodes "preprod-k8s-cluster-node-8": <nil>
E0929 15:35:35.197149       1 routecontroller.go:197] Error updating node preprod-k8s-cluster-node-8, retrying: Operation cannot be fulfilled on nodes "preprod-k8s-cluster-node-8": <nil>
// Controller-Manager hitting the maxWaitForUnmountDuration
I0929 15:37:53.573132       1 reconciler.go:138] Started DetachVolume for volume "kubernetes.io/azure-disk/mongo-disk-1" from node "preprod-k8s-cluster-node-2". This volume is not safe to detach, but maxWaitForUnmountDuration 6m0s expired, force detaching
I0929 15:37:59.699792       1 reconciler.go:138] Started DetachVolume for volume "kubernetes.io/azure-disk/mongo-disk-2" from node "preprod-k8s-cluster-node-2". This volume is not safe to detach, but maxWaitForUnmountDuration 6m0s expired, force detaching
I0929 15:38:00.009523       1 reconciler.go:138] Started DetachVolume for volume "kubernetes.io/azure-disk/mongo-disk-3" from node "preprod-k8s-cluster-node-6". This volume is not safe to detach, but maxWaitForUnmountDuration 6m0s expired, force detaching
// Detach Success 
I0929 15:39:53.978423       1 operation_executor.go:595] DetachVolume.Detach succeeded for volume "kubernetes.io/azure-disk/mongo-disk-1" (spec.Name: "mongo-persistent-storage") from node "preprod-k8s-cluster-node-2".
I0929 15:39:59.982579       1 operation_executor.go:595] DetachVolume.Detach succeeded for volume "kubernetes.io/azure-disk/mongo-disk-2" (spec.Name: "mongo-persistent-storage") from node "preprod-k8s-cluster-node-2".
I0929 15:40:00.284553       1 operation_executor.go:595] DetachVolume.Detach succeeded for volume "kubernetes.io/azure-disk/mongo-disk-3" (spec.Name: "mongo-persistent-storage") from node "preprod-k8s-cluster-node-6".
I0929 15:40:30.146532       1 replication_controller.go:323] Observed updated replication controller mongo-1. Desired pod count change: 1->1
I0929 15:40:30.146682       1 replication_controller.go:323] Observed updated replication controller mongo-2. Desired pod count change: 1->1
I0929 15:40:30.146701       1 replication_controller.go:323] Observed updated replication controller influxdb-grafana. Desired pod count change: 1->1
I0929 15:40:30.146715       1 replication_controller.go:323] Observed updated replication controller jenkins. Desired pod count change: 1->1
I0929 15:40:30.153460       1 replication_controller.go:323] Observed updated replication controller mongo-3. Desired pod count change: 1->1
I0929 15:40:30.153503       1 replication_controller.go:323] Observed updated replication controller heapster. Desired pod count change: 1->1
I0929 15:40:30.153515       1 replication_controller.go:323] Observed updated replication controller kube-dns-v19. Desired pod count change: 2->2
I0929 15:40:41.052039       1 replication_controller.go:514] Too few "integration"/"mongo-1" replicas, need 1, creating 1
I0929 15:40:41.057729       1 event.go:217] Event(api.ObjectReference{Kind:"ReplicationController", Namespace:"integration", Name:"mongo-1", UID:"139b85a5-865b-11e6-b6aa-000d3ab30bf6", APIVersion:"v1", ResourceVersion:"22431", FieldPath:""}): type: 'Normal' reason: 'SuccessfulCreate' Created pod: mongo-1-nneoq
// Recreating the Replication Controllers [mongo-1, mongo-2, mongo-3]
I0929 15:40:41.068339       1 replication_controller.go:323] Observed updated replication controller mongo-1. Desired pod count change: 1->1
I0929 15:40:41.080848       1 replication_controller.go:323] Observed updated replication controller mongo-1. Desired pod count change: 1->1
I0929 15:40:41.088741       1 replication_controller.go:514] Too few "integration"/"mongo-2" replicas, need 1, creating 1
I0929 15:40:41.093678       1 event.go:217] Event(api.ObjectReference{Kind:"ReplicationController", Namespace:"integration", Name:"mongo-2", UID:"13a15396-865b-11e6-b6aa-000d3ab30bf6", APIVersion:"v1", ResourceVersion:"22439", FieldPath:""}): type: 'Normal' reason: 'SuccessfulCreate' Created pod: mongo-2-8o6jr
I0929 15:40:41.103261       1 replication_controller.go:323] Observed updated replication controller mongo-2. Desired pod count change: 1->1
I0929 15:40:41.111057       1 replication_controller.go:323] Observed updated replication controller mongo-2. Desired pod count change: 1->1
I0929 15:40:41.122912       1 replication_controller.go:514] Too few "integration"/"mongo-3" replicas, need 1, creating 1
I0929 15:40:41.128488       1 event.go:217] Event(api.ObjectReference{Kind:"ReplicationController", Namespace:"integration", Name:"mongo-3", UID:"13a68b99-865b-11e6-b6aa-000d3ab30bf6", APIVersion:"v1", ResourceVersion:"22447", FieldPath:""}): type: 'Normal' reason: 'SuccessfulCreate' Created pod: mongo-3-zt694
I0929 15:40:41.133833       1 replication_controller.go:323] Observed updated replication controller mongo-3. Desired pod count change: 1->1
I0929 15:40:41.145952       1 replication_controller.go:323] Observed updated replication controller mongo-3. Desired pod count change: 1->1
// Attaching Volumes
I0929 15:40:41.168196       1 reconciler.go:168] Started AttachVolume for volume "kubernetes.io/azure-disk/mongo-disk-1" to node "preprod-k8s-cluster-node-2"
I0929 15:40:41.168297       1 reconciler.go:168] Started AttachVolume for volume "kubernetes.io/azure-disk/mongo-disk-3" to node "preprod-k8s-cluster-node-5"
I0929 15:40:41.168318       1 reconciler.go:168] Started AttachVolume for volume "kubernetes.io/azure-disk/mongo-disk-2" to node "preprod-k8s-cluster-node-7"
I0929 15:41:12.600560       1 replication_controller.go:323] Observed updated replication controller mongo-3. Desired pod count change: 1->1
I0929 15:42:07.431348       1 replication_controller.go:323] Observed updated replication controller mongo-1. Desired pod count change: 1->1
W0929 15:42:40.356874       1 reflector.go:330] pkg/controller/garbagecollector/garbagecollector.go:758: watch of <nil> ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [21873/20930]) [22872]
// Attach Success
I0929 15:42:41.640471       1 operation_executor.go:515] AttachVolume.Attach succeeded for volume "kubernetes.io/azure-disk/mongo-disk-1" (spec.Name: "mongo-persistent-storage") from node "preprod-k8s-cluster-node-2".
I0929 15:42:41.651637       1 operation_executor.go:515] AttachVolume.Attach succeeded for volume "kubernetes.io/azure-disk/mongo-disk-2" (spec.Name: "mongo-persistent-storage") from node "preprod-k8s-cluster-node-7".
I0929 15:42:41.677058       1 operation_executor.go:515] AttachVolume.Attach succeeded for volume "kubernetes.io/azure-disk/mongo-disk-3" (spec.Name: "mongo-persistent-storage") from node "preprod-k8s-cluster-node-5".