rook-agent format volume

mongoDB pod

apiVersion: v1
kind: Pod
metadata:
  annotations:
    kubernetes.io/created-by: |
      {"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicaSet","namespace":"coyo-production","name":"coyo-production-mongodb-6d55cbfb64","uid":"bf429dbc-2179-11e8-8fed-0201bc1fae81","apiVersion":"extensions","resourceVersion":"11182523"}}
  creationTimestamp: 2018-03-07T00:22:36Z
  generateName: coyo-production-mongodb-6d55cbfb64-
  labels:
    app: coyo-production-mongodb
    pod-template-hash: "2811769620"
  name: coyo-production-mongodb-6d55cbfb64-bqhjm
  namespace: coyo-production
  ownerReferences:
  - apiVersion: extensions/v1beta1
    blockOwnerDeletion: true
    controller: true
    kind: ReplicaSet
    name: coyo-production-mongodb-6d55cbfb64
    uid: bf429dbc-2179-11e8-8fed-0201bc1fae81
  resourceVersion: "11182701"
  selfLink: /api/v1/namespaces/coyo-production/pods/coyo-production-mongodb-6d55cbfb64-bqhjm
  uid: a34e0e3f-219d-11e8-8a76-0201bc1fae81
spec:
  containers:
  - image: mongo:3.2.19
    imagePullPolicy: IfNotPresent
    livenessProbe:
      exec:
        command:
        - mongo
        - --eval
        - db.adminCommand('ping')
      failureThreshold: 3
      initialDelaySeconds: 30
      periodSeconds: 10
      successThreshold: 1
      timeoutSeconds: 5
    name: coyo-production-mongodb
    ports:
    - containerPort: 27017
      name: mongodb
      protocol: TCP
    readinessProbe:
      exec:
        command:
        - mongo
        - --eval
        - db.adminCommand('ping')
      failureThreshold: 3
      initialDelaySeconds: 5
      periodSeconds: 10
      successThreshold: 1
      timeoutSeconds: 1
    resources:
      limits:
        cpu: "8"
        memory: 24Gi
      requests:
        cpu: "1"
        memory: 2Gi
    terminationMessagePath: /dev/termination-log
    terminationMessagePolicy: File
    volumeMounts:
    - mountPath: /data/db
      name: data
    - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
      name: default-token-sn7tg
      readOnly: true
  dnsPolicy: ClusterFirst
  nodeName: k8s-node-2
  restartPolicy: Always
  schedulerName: default-scheduler
  securityContext: {}
  serviceAccount: default
  serviceAccountName: default
  terminationGracePeriodSeconds: 30
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: coyo-production-mongodb
  - name: default-token-sn7tg
    secret:
      defaultMode: 420
      secretName: default-token-sn7tg
status:
  conditions:
  - lastProbeTime: null
    lastTransitionTime: 2018-03-07T00:22:36Z
    status: "True"
    type: Initialized
  - lastProbeTime: null
    lastTransitionTime: 2018-03-07T00:23:55Z
    status: "True"
    type: Ready
  - lastProbeTime: null
    lastTransitionTime: 2018-03-07T00:22:36Z
    status: "True"
    type: PodScheduled
  containerStatuses:
  - containerID: docker://bdce229de20f1937d00c015a31e18a26d4484fa68a6e0b0ada0aee486d154833
    image: mongo:3.2.19
    imageID: docker-pullable://mongo@sha256:b0ffb38cf6d9479a2b099b159926d72ebfec94dc206b6d5647a034f46f212de2
    lastState: {}
    name: coyo-production-mongodb
    ready: true
    restartCount: 0
    state:
      running:
        startedAt: 2018-03-07T00:23:43Z
  hostIP: 194.64.102.7
  phase: Running
  podIP: 10.233.83.119
  qosClass: Burstable
  startTime: 2018-03-07T00:22:36Z

mongoDB PVC

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"abfed267-fc5b-11e7-ba1a-aa996137dbd1","leaseDurationSeconds":15,"acquireTime":"2018-01-24T09:14:48Z","renewTime":"2018-01-24T09:15:15Z","leaderTransitions":0}'
    pv.kubernetes.io/bind-completed: "yes"
    pv.kubernetes.io/bound-by-controller: "yes"
    volume.beta.kubernetes.io/storage-provisioner: rook.io/block
  creationTimestamp: 2018-01-24T09:14:48Z
  name: coyo-production-mongodb
  namespace: coyo-production
  resourceVersion: "3870791"
  selfLink: /api/v1/namespaces/coyo-production/persistentvolumeclaims/coyo-production-mongodb
  uid: 06cbae31-00e7-11e8-a8f1-0201fa246d34
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 2Ti
  storageClassName: generic
  volumeName: pvc-06cbae31-00e7-11e8-a8f1-0201fa246d34
status:
  accessModes:
  - ReadWriteOnce
  capacity:
    storage: 2Ti
  phase: Bound

rook-agent.log

2018-03-06 22:25:16.102470 I | flexdriver: calling agent to attach volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:25:16.130557 I | flexvolume: Creating Volume attach Resource cluster-base/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34: {Image:pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 Pool:generic ClusterName:cluster-base StorageClass:generic MountDir:/var/lib/kubelet/pods/f4fa400a-218c-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 FsName: Path: RW:rw FsType:ext4 VolumeName:pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 Pod:coyo-production-postgresql-57d9d894c5-ndml4 PodID:f4fa400a-218c-11e8-8a76-0201bc1fae81 PodNamespace:coyo-production}
2018-03-06 22:25:16.143544 I | ceph-volumeattacher: attaching volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 cluster cluster-base
2018-03-06 22:25:16.153118 I | cephmon: parsing mon endpoints: rook-ceph-mon5=10.233.20.175:6790,rook-ceph-mon2=10.233.30.225:6790,rook-ceph-mon3=10.233.48.238:6790
2018-03-06 22:25:16.153319 I | op-mon: loaded: maxMonID=5, mons=map[rook-ceph-mon3:0xc4202ade80 rook-ceph-mon5:0xc4202adce0 rook-ceph-mon2:0xc4202ade40], mapping=&{Node:map[rook-ceph-mon2:0xc4202adea0 rook-ceph-mon3:0xc4202adee0 rook-ceph-mon5:0xc4202adf40] Port:map[]}
2018-03-06 22:25:16.154184 I | exec: Running command: rbd map generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 --id admin --cluster=cluster-base --keyring=/tmp/cluster-base.keyring101532956 -m 10.233.20.175:6790,10.233.30.225:6790,10.233.48.238:6790 --conf=/dev/null
2018-03-06 22:25:18.305554 I | flexdriver: 
2018-03-06 22:25:18.306495 I | flexdriver: Ignore error about Mount failed: exit status 32. Kubernetes does this to check whether the volume has been formatted. It will format and retry again. https://github.com/kubernetes/kubernetes/blob/release-1.7/pkg/util/mount/mount_linux.go#L360
2018-03-06 22:25:18.306791 I | flexdriver: formatting volume pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 devicePath /dev/rbd4 deviceMountPath /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 fs ext4 with options [rw]
2018-03-06 22:25:18.307054 I | flexdriver: mounting global mount path /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 on /var/lib/kubelet/pods/f4fa400a-218c-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:25:18.403049 I | flexdriver: 
2018-03-06 22:25:18.403421 I | flexdriver: volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 has been attached and mounted
ERROR: logging before flag.Parse: W0306 22:26:19.262302       7 reflector.go:334] github.com/rook/rook/vendor/github.com/rook/operator-kit/watcher.go:76: watch of *v1alpha1.Cluster ended with: too old resource version: 11167730 (11168828)
2018-03-06 22:26:25.047877 I | flexdriver: unmounting mount dir: /var/lib/kubelet/pods/f4fa400a-218c-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:26:25.189932 I | flexvolume: Deleting attachment for mountDir /var/lib/kubelet/pods/f4fa400a-218c-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 from Volume attach CRD cluster-base/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:26:25.209205 I | rook-agent-crd: updated Volumeattach CRD pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:26:25.240220 I | flexdriver: calling agent to attach volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:26:25.256531 I | rook-agent-crd: updated Volumeattach CRD pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:26:25.257526 I | ceph-volumeattacher: volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 is already attached. The device path is /dev/rbd4
2018-03-06 22:26:25.480336 I | flexdriver: 
2018-03-06 22:26:25.481258 I | flexdriver: calling agent to detach mountDir: /var/lib/kubelet/pods/f4fa400a-218c-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:26:25.482528 I | ceph-volumeattacher: detaching volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 cluster cluster-base
2018-03-06 22:26:25.579617 I | cephmon: parsing mon endpoints: rook-ceph-mon5=10.233.20.175:6790,rook-ceph-mon2=10.233.30.225:6790,rook-ceph-mon3=10.233.48.238:6790
2018-03-06 22:26:25.579807 I | op-mon: loaded: maxMonID=5, mons=map[rook-ceph-mon5:0xc420908b00 rook-ceph-mon2:0xc420908b40 rook-ceph-mon3:0xc420908b80], mapping=&{Node:map[rook-ceph-mon2:0xc420908ba0 rook-ceph-mon3:0xc420908be0 rook-ceph-mon5:0xc420908c20] Port:map[]}
2018-03-06 22:26:25.582621 I | exec: Running command: rbd unmap generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 --id admin --cluster=cluster-base --keyring=/tmp/cluster-base.keyring237427659 -m 10.233.20.175:6790,10.233.30.225:6790,10.233.48.238:6790 --conf=/dev/null
2018-03-06 22:26:25.964645 E | flexdriver: Detach volume from /var/lib/kubelet/pods/f4fa400a-218c-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 failed: Failed to detach volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34: failed to detach volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 cluster cluster-base. failed to unmap image generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34: Failed to complete rbd: exit status 16. output: rbd: run_cmd(udevadm): exited with status 127
rbd: sysfs write failed
rbd: unmap failed: (16) Device or resource busy
2018-03-06 22:29:24.404323 I | flexdriver: ERROR: logging before flag.Parse: E0306 22:26:25.858718   51246 mount_linux.go:140] Mount failed: exit status 32
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 --scope -- mount -t ext4 -o rw,defaults /dev/rbd4 /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
Output: Running scope as unit run-r05bdd3d4c1124444bf709cde35ff3def.scope.
mount: mount point /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 does not exist

ERROR: logging before flag.Parse: I0306 22:26:25.865635   51246 mount_linux.go:404] Disk "/dev/rbd4" appears to be unformatted, attempting to format as type: "ext4" with options: [-F /dev/rbd4]
ERROR: logging before flag.Parse: I0306 22:29:24.347948   51246 mount_linux.go:408] Disk successfully formatted (mkfs): ext4 - /dev/rbd4 /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
ERROR: logging before flag.Parse: E0306 22:29:24.399719   51246 mount_linux.go:140] Mount failed: exit status 32
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 --scope -- mount -t ext4 -o rw,defaults /dev/rbd4 /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
Output: Running scope as unit run-rf8a82b937309425fb2b5aa41888b5461.scope.
mount: mount point /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 does not exist

2018-03-06 22:29:24.418058 E | flexdriver: mount volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 failed: failed to mount volume /dev/rbd4 [ext4] to /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34, error mount failed: exit status 32
Mounting command: systemd-run
Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 --scope -- mount -t ext4 -o rw,defaults /dev/rbd4 /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
Output: Running scope as unit run-rf8a82b937309425fb2b5aa41888b5461.scope.
mount: mount point /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 does not exist

2018-03-06 22:29:25.234662 I | flexdriver: calling agent to attach volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:29:25.261390 I | ceph-volumeattacher: volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 is already attached. The device path is /dev/rbd4
2018-03-06 22:29:27.303636 I | flexdriver: 
2018-03-06 22:29:27.307779 I | flexdriver: Ignore error about Mount failed: exit status 32. Kubernetes does this to check whether the volume has been formatted. It will format and retry again. https://github.com/kubernetes/kubernetes/blob/release-1.7/pkg/util/mount/mount_linux.go#L360
2018-03-06 22:29:27.310017 I | flexdriver: formatting volume pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 devicePath /dev/rbd4 deviceMountPath /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 fs ext4 with options [rw]
2018-03-06 22:29:27.310347 I | flexdriver: mounting global mount path /var/lib/kubelet/plugins/rook.io/rook/mounts/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 on /var/lib/kubelet/pods/6794e04f-218d-11e8-8a76-0201bc1fae81/volumes/rook.io~rook/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34
2018-03-06 22:29:27.440471 I | flexdriver: 
2018-03-06 22:29:27.448435 I | flexdriver: volume generic/pvc-5333da36-fcf5-11e7-9f4a-0201fa246d34 has been attached and mounted

StorageClass

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
  creationTimestamp: 2018-01-08T14:11:51Z
  labels:
    app: cluster-base-storage
    chart: storage-0.1.0
    heritage: Tiller
    release: cluster-base
  name: generic
  resourceVersion: "1033206"
  selfLink: /apis/storage.k8s.io/v1/storageclasses/generic
  uid: df9c4742-f47d-11e7-b353-0201fa246d34
parameters:
  clusterName: cluster-base
  fstype: ext4
  pool: generic
provisioner: rook.io/block
reclaimPolicy: Delete