Useful Kubernetes commands and their respective output List pods from the default namespace ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get pods
NAME READY STATUS RESTARTS AGE
signalfx-agent-d9p55 1/1 Running 0 52m
signalfx-agent-kctbx 1/1 Running 0 52m
signalfx-agent-qtll8 1/1 Running 0 52m
List pods from all namespaces (including system pods) ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get pods --all-namespaces
NAMESPACE NAME READY STATUS RESTARTS AGE
default signalfx-agent-d9p55 1/1 Running 0 53m
default signalfx-agent-kctbx 1/1 Running 0 53m
default signalfx-agent-qtll8 1/1 Running 0 53m
kube-system calico-etcd-55lj6 1/1 Running 1 4d
kube-system calico-kube-controllers-685755779f-mpsrn 1/1 Running 1 4d
kube-system calico-node-2vpkn 2/2 Running 2 4d
kube-system calico-node-k4dqs 2/2 Running 3 4d
kube-system calico-node-scgq4 2/2 Running 4 4d
kube-system etcd-centos-kubernetes-01.localdomain 1/1 Running 1 4d
kube-system kube-apiserver-centos-kubernetes-01.localdomain 1/1 Running 1 4d
kube-system kube-controller-manager-centos-kubernetes-01.localdomain 1/1 Running 1 4d
kube-system kube-dns-86f4d74b45-th7zn 3/3 Running 3 2h
kube-system kube-proxy-fwl98 1/1 Running 1 4d
kube-system kube-proxy-gf7gq 1/1 Running 1 4d
kube-system kube-proxy-hqw79 1/1 Running 1 4d
kube-system kube-scheduler-centos-kubernetes-01.localdomain 1/1 Running 1 4d
╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get serviceAccounts
NAME SECRETS AGE
default 1 4d
signalfx-agent 1 6m
╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get clusterroles
NAME AGE
admin 6d
calico-cni-plugin 6d
calico-kube-controllers 6d
cluster-admin 6d
edit 6d
signalfx-agent 6d
system:aggregate-to-admin 6d
system:aggregate-to-edit 6d
system:aggregate-to-view 6d
system:auth-delegator 6d
system:aws-cloud-provider 6d
system:basic-user 6d
system:certificates.k8s.io:certificatesigningrequests:nodeclient 6d
system:certificates.k8s.io:certificatesigningrequests:selfnodeclient 6d
system:controller:attachdetach-controller 6d
system:controller:certificate-controller 6d
system:controller:clusterrole-aggregation-controller 6d
system:controller:cronjob-controller 6d
system:controller:daemon-set-controller 6d
system:controller:deployment-controller 6d
system:controller:disruption-controller 6d
system:controller:endpoint-controller 6d
system:controller:generic-garbage-collector 6d
system:controller:horizontal-pod-autoscaler 6d
system:controller:job-controller 6d
system:controller:namespace-controller 6d
system:controller:node-controller 6d
system:controller:persistent-volume-binder 6d
system:controller:pod-garbage-collector 6d
system:controller:pv-protection-controller 6d
system:controller:pvc-protection-controller 6d
system:controller:replicaset-controller 6d
system:controller:replication-controller 6d
system:controller:resourcequota-controller 6d
system:controller:route-controller 6d
system:controller:service-account-controller 6d
system:controller:service-controller 6d
system:controller:statefulset-controller 6d
system:controller:ttl-controller 6d
system:discovery 6d
system:heapster 6d
system:kube-aggregator 6d
system:kube-controller-manager 6d
system:kube-dns 6d
system:kube-scheduler 6d
system:kubelet-api-admin 6d
system:node 6d
system:node-bootstrapper 6d
system:node-problem-detector 6d
system:node-proxier 6d
system:persistent-volume-provisioner 6d
system:volume-scheduler 6d
view 6d
╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get serviceaccounts/signalfx-agent -o yaml
apiVersion: v1
kind: ServiceAccount
metadata:
creationTimestamp: 2018-05-01T23:12:45Z
name: signalfx-agent
namespace: default
resourceVersion: "13295"
selfLink: /api/v1/namespaces/default/serviceaccounts/signalfx-agent
uid: 2881601d-4d95-11e8-bef6-000c2977ae1c
secrets:
- name: signalfx-agent-token-zfv9d
╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get clusterroles/signalfx-agent -o yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"rbac.authorization.k8s.io/v1","kind":"ClusterRole","metadata":{"annotations":{},"labels":{"app":"signalfx-agent"},"name":"signalfx-agent","namespace":""},"rules":[{"apiGroups":[""],"resources":["events","namespaces","namespaces/status","nodes","nodes/spec","pods","pods/status","replicationcontrollers","replicationcontrollers/status","services"],"verbs":["get","list","watch"]},{"apiGroups":[""],"resources":["configmaps"],"verbs":["get","update","create"]},{"apiGroups":[""],"resources":["nodes/stats"],"verbs":["get","list","create"]},{"apiGroups":["apps"],"resources":["daemonsets","deployments","replicasets","statefulsets"],"verbs":["get","list","watch"]},{"apiGroups":["extensions"],"resources":["daemonsets","deployments","replicasets"],"verbs":["get","list","watch"]}]}
creationTimestamp: 2018-05-03T23:32:40Z
labels:
app: signalfx-agent
name: signalfx-agent
resourceVersion: "245411"
selfLink: /apis/rbac.authorization.k8s.io/v1/clusterroles/signalfx-agent
uid: 45089cfb-4f2a-11e8-bef6-000c2977ae1c
rules:
- apiGroups:
- ""
resources:
- events
- namespaces
- namespaces/status
- nodes
- nodes/spec
- pods
- pods/status
- replicationcontrollers
- replicationcontrollers/status
- services
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- update
- create
- apiGroups:
- ""
resources:
- nodes/stats
verbs:
- get
- list
- create
- apiGroups:
- apps
resources:
- daemonsets
- deployments
- replicasets
- statefulsets
verbs:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- daemonsets
- deployments
- replicasets
verbs:
- get
- list
- watch
Dump the SignalFx Agent cluster role binding ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get clusterrolebinding/signalfx-agent -o yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
creationTimestamp: 2018-05-04T16:31:35Z
name: signalfx-agent
resourceVersion: "4391"
selfLink: /apis/rbac.authorization.k8s.io/v1/clusterrolebindings/signalfx-agent
uid: 9c70f63c-4fb8-11e8-a417-000c2977ae1c
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: signalfx-agent
namespace: default
Dump the SignalFx agent configmap ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get configmap/signalfx-agent -o yaml
apiVersion: v1
data:
agent.yaml: |
signalFxAccessToken: ${SFX_ACCESS_TOKEN}
intervalSeconds: 15
logging:
level: info
globalDimensions:
kubernetes_cluster: SECRET-SQUIRREL
sendMachineID: true
observers:
- type: k8s-api
monitors:
- type: collectd/cpu
- type: collectd/cpufreq
- type: collectd/df
hostFSPath: /hostfs
- type: collectd/disk
- type: collectd/interface
- type: collectd/load
- type: collectd/memory
- type: collectd/protocols
- type: collectd/signalfx-metadata
procFSPath: /hostfs/proc
etcPath: /hostfs/etc
persistencePath: /run
- type: collectd/uptime
- type: collectd/vmem
- type: collectd/processes
processes:
- collectd
- signalfx-agent
- type: kubelet-stats
kubeletAPI:
authType: serviceAccount
skipVerify: true
# Collects k8s cluster-level metrics
- type: kubernetes-cluster
- type: docker-container-stats
dockerURL: unix:///var/run/docker.sock
excludedImages:
- '*pause-amd64*'
labelsToDimensions:
io.kubernetes.container.name: container_spec_name
io.kubernetes.pod.name: kubernetes_pod_name
io.kubernetes.pod.uid: kubernetes_pod_uid
io.kubernetes.pod.namespace: kubernetes_namespace
metricsToExclude:
- metricNames:
- 'cpu.usage*'
- 'cpu.percent'
- 'memory.usage*'
- 'memory.percent'
- 'blkio.io_service_bytes_recursive.*'
negated: true
- type: collectd/activemq
discoveryRule: container_image =~ "activemq" && private_port == 1099
- type: collectd/apache
discoveryRule: container_image =~ "apache" && private_port == 80
- type: collectd/cassandra
discoveryRule: container_image =~ "cassandra" && private_port == 7199
- type: collectd/consul
discoveryRule: container_image =~ "consul" && private_port == 8500
- type: collectd/elasticsearch
discoveryRule: container_image =~ "elasticsearch" && port == 9200
- type: collectd/etcd
discoveryRule: container_image =~ "etcd" && port == 2379
clusterName: my-cluster
- type: collectd/haproxy
discoveryRule: container_image =~ "haproxy" && port == 9000
- type: collectd/kafka
discoveryRule: container_image =~ "kafka" && private_port == 9092
- type: collectd/memcached
discoveryRule: container_image =~ "memcache" && private_port == 11211
- type: collectd/mongodb
discoveryRule: container_image =~ "mongo" && private_port == 27017
databases:
- mydatabase
- type: collectd/mysql
discoveryRule: container_image =~ "mysql" && private_port == 3306
databases:
- name: mydb
username: admin
- type: collectd/nginx
discoveryRule: container_image =~ "nginx" && private_port == 80
- type: collectd/rabbitmq
discoveryRule: container_image =~ "rabbitmq" && private_port == 15672
- type: collectd/redis
discoveryRule: container_image =~ "redis" && private_port == 6379
- type: collectd/spark
discoveryRule: container_image =~ "spark" && private_port == 8080
clusterType: Standalone
collectApplicationMetrics: true
isMaster: true
- type: collectd/spark
discoveryRule: container_image =~ "spark" && private_port >= 8081
clusterType: Standalone
isMaster: false
- type: collectd/zookeeper
discoveryRule: container_image =~ "zookeeper" && private_port == 2181
collectd:
readThreads: 5
writeQueueLimitHigh: 500000
writeQueueLimitLow: 400000
timeout: 40
logLevel: info
metricsToExclude:
- metricNames:
- "container_cpu_user_seconds_total"
- "container_cpu_system_seconds_total"
- "container_cpu_usage_seconds_total"
- "container_cpu_utilization_per_core"
- "container_fs_reads_total"
- "container_fs_sector_reads_total"
- "container_fs_reads_merged_total"
- "container_fs_read_seconds_total"
- "container_fs_writes_total"
- "container_fs_sector_writes_total"
- "container_fs_writes_merged_total"
- "container_fs_write_seconds_total"
- "container_fs_io_current"
- "container_fs_io_time_seconds_total"
- "container_fs_io_time_weighted_seconds_total"
- "container_last_seen"
- "container_tasks_state"
- "pod_network_receive_packets_total"
- "pod_network_receive_packets_dropped_total"
- "pod_network_transmit_packets_total"
- "pod_network_transmit_packets_dropped_total"
- "machine_cpu_frequency_khz"
kind: ConfigMap
metadata:
creationTimestamp: 2018-05-04T16:09:12Z
labels:
app: signalfx-agent
name: signalfx-agent
namespace: default
resourceVersion: "2562"
selfLink: /api/v1/namespaces/default/configmaps/signalfx-agent
uid: 7c078fba-4fb5-11e8-a417-000c2977ae1c
Dump the SignalFx agent daemonset ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl get daemonset/signalfx-agent -o yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
creationTimestamp: 2018-05-04T16:32:24Z
generation: 1
labels:
app: signalfx-agent
version: 3.1.2
name: signalfx-agent
namespace: default
resourceVersion: "4542"
selfLink: /apis/extensions/v1beta1/namespaces/default/daemonsets/signalfx-agent
uid: b989aa6f-4fb8-11e8-a417-000c2977ae1c
spec:
revisionHistoryLimit: 10
selector:
matchLabels:
app: signalfx-agent
template:
metadata:
creationTimestamp: null
labels:
app: signalfx-agent
version: 3.1.2
spec:
containers:
- command:
- /bin/signalfx-agent
env:
- name: SFX_ACCESS_TOKEN
valueFrom:
secretKeyRef:
key: access-token
name: signalfx-agent
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: MY_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: MY_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
image: quay.io/signalfx/signalfx-agent:3.1.2
imagePullPolicy: IfNotPresent
name: signalfx-agent
resources: {}
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/signalfx
name: config
- mountPath: /etc/machine-id
name: machine-id
readOnly: true
- mountPath: /hostfs
name: hostfs
readOnly: true
- mountPath: /var/run/docker.sock
name: docker
readOnly: true
dnsPolicy: ClusterFirst
hostNetwork: true
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: signalfx-agent
serviceAccountName: signalfx-agent
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoSchedule
key: node.alpha.kubernetes.io/role
operator: Exists
- effect: NoSchedule
key: node-role.kubernetes.io/master
operator: Exists
volumes:
- configMap:
defaultMode: 420
name: signalfx-agent
name: config
- hostPath:
path: /
type: ""
name: hostfs
- hostPath:
path: /var/run/docker.sock
type: ""
name: docker
- hostPath:
path: /etc/machine-id
type: ""
name: machine-id
templateGeneration: 1
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
status:
currentNumberScheduled: 3
desiredNumberScheduled: 3
numberAvailable: 3
numberMisscheduled: 0
numberReady: 3
observedGeneration: 1
updatedNumberScheduled: 3
Get detailed information about the SignalFx Agent daemonset ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl describe daemonset.apps/signalfx-agent
Name: signalfx-agent
Selector: app=signalfx-agent
Node-Selector: <none>
Labels: app=signalfx-agent
version=3.1.2
Annotations: <none>
Desired Number of Nodes Scheduled: 3
Current Number of Nodes Scheduled: 3
Number of Nodes Scheduled with Up-to-date Pods: 3
Number of Nodes Scheduled with Available Pods: 3
Number of Nodes Misscheduled: 0
Pods Status: 3 Running / 0 Waiting / 0 Succeeded / 0 Failed
Pod Template:
Labels: app=signalfx-agent
version=3.1.2
Service Account: signalfx-agent
Containers:
signalfx-agent:
Image: quay.io/signalfx/signalfx-agent:3.1.2
Port: <none>
Host Port: <none>
Command:
/bin/signalfx-agent
Environment:
SFX_ACCESS_TOKEN: <set to the key 'access-token' in secret 'signalfx-agent'> Optional: false
MY_POD_NAME: (v1:metadata.name)
MY_NODE_NAME: (v1:spec.nodeName)
MY_NAMESPACE: (v1:metadata.namespace)
Mounts:
/etc/machine-id from machine-id (ro)
/etc/signalfx from config (rw)
/hostfs from hostfs (ro)
/var/run/docker.sock from docker (ro)
Volumes:
config:
Type: ConfigMap (a volume populated by a ConfigMap)
Name: signalfx-agent
Optional: false
hostfs:
Type: HostPath (bare host directory volume)
Path: /
HostPathType:
docker:
Type: HostPath (bare host directory volume)
Path: /var/run/docker.sock
HostPathType:
machine-id:
Type: HostPath (bare host directory volume)
Path: /etc/machine-id
HostPathType:
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal SuccessfulCreate 37m daemonset-controller Created pod: signalfx-agent-qtll8
Normal SuccessfulCreate 37m daemonset-controller Created pod: signalfx-agent-d9p55
Normal SuccessfulCreate 37m daemonset-controller Created pod: signalfx-agent-kctbx
Get detailed information about nodes in the cluster ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl describe nodes
Name: centos-kubernetes-01.localdomain
Roles: master
Labels: beta.kubernetes.io/arch=amd64
beta.kubernetes.io/os=linux
kubernetes.io/hostname=centos-kubernetes-01.localdomain
node-role.kubernetes.io/master=
Annotations: node.alpha.kubernetes.io/ttl=0
volumes.kubernetes.io/controller-managed-attach-detach=true
CreationTimestamp: Fri, 27 Apr 2018 12:28:24 -0500
Taints: node-role.kubernetes.io/master:NoSchedule
Unschedulable: false
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
OutOfDisk False Tue, 01 May 2018 18:52:59 -0500 Fri, 27 Apr 2018 12:28:17 -0500 KubeletHasSufficientDisk kubelet has sufficient disk space available
MemoryPressure False Tue, 01 May 2018 18:52:59 -0500 Fri, 27 Apr 2018 12:28:17 -0500 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Tue, 01 May 2018 18:52:59 -0500 Fri, 27 Apr 2018 12:28:17 -0500 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Tue, 01 May 2018 18:52:59 -0500 Fri, 27 Apr 2018 12:28:17 -0500 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Tue, 01 May 2018 18:52:59 -0500 Fri, 27 Apr 2018 12:30:14 -0500 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.1.115
Hostname: centos-kubernetes-01.localdomain
Capacity:
cpu: 4
ephemeral-storage: 17394Mi
hugepages-2Mi: 0
memory: 3873244Ki
pods: 110
Allocatable:
cpu: 4
ephemeral-storage: 16415037823
hugepages-2Mi: 0
memory: 3770844Ki
pods: 110
System Info:
Machine ID: 926b0ae3e78a4d09bd6ab021eb96891e
System UUID: 564DA426-F2DE-E9A7-8944-3DB8A377AE1C
Boot ID: a8a5980b-5073-477a-b3f2-01ac01536f4b
Kernel Version: 3.10.0-693.21.1.el7.x86_64
OS Image: CentOS Linux 7 (Core)
Operating System: linux
Architecture: amd64
Container Runtime Version: docker://1.13.1
Kubelet Version: v1.10.2
Kube-Proxy Version: v1.10.2
PodCIDR: 10.244.0.0/24
ExternalID: centos-kubernetes-01.localdomain
Non-terminated Pods: (10 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits
--------- ---- ------------ ---------- --------------- -------------
default signalfx-agent-qtll8 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system calico-etcd-55lj6 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system calico-kube-controllers-685755779f-mpsrn 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system calico-node-2vpkn 250m (6%) 0 (0%) 0 (0%) 0 (0%)
kube-system etcd-centos-kubernetes-01.localdomain 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system kube-apiserver-centos-kubernetes-01.localdomain 250m (6%) 0 (0%) 0 (0%) 0 (0%)
kube-system kube-controller-manager-centos-kubernetes-01.localdomain 200m (5%) 0 (0%) 0 (0%) 0 (0%)
kube-system kube-dns-86f4d74b45-th7zn 260m (6%) 0 (0%) 110Mi (2%) 170Mi (4%)
kube-system kube-proxy-fwl98 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system kube-scheduler-centos-kubernetes-01.localdomain 100m (2%) 0 (0%) 0 (0%) 0 (0%)
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
CPU Requests CPU Limits Memory Requests Memory Limits
------------ ---------- --------------- -------------
1060m (26%) 0 (0%) 110Mi (2%) 170Mi (4%)
Events: <none>
Name: centos-kubernetes-02.localdomain
Roles: <none>
Labels: beta.kubernetes.io/arch=amd64
beta.kubernetes.io/os=linux
kubernetes.io/hostname=centos-kubernetes-02.localdomain
Annotations: node.alpha.kubernetes.io/ttl=0
volumes.kubernetes.io/controller-managed-attach-detach=true
CreationTimestamp: Fri, 27 Apr 2018 12:30:12 -0500
Taints: <none>
Unschedulable: false
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
OutOfDisk False Tue, 01 May 2018 18:53:00 -0500 Tue, 01 May 2018 17:11:53 -0500 KubeletHasSufficientDisk kubelet has sufficient disk space available
MemoryPressure False Tue, 01 May 2018 18:53:00 -0500 Tue, 01 May 2018 17:11:53 -0500 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Tue, 01 May 2018 18:53:00 -0500 Tue, 01 May 2018 17:11:53 -0500 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Tue, 01 May 2018 18:53:00 -0500 Fri, 27 Apr 2018 12:30:12 -0500 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Tue, 01 May 2018 18:53:00 -0500 Tue, 01 May 2018 17:12:03 -0500 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.1.116
Hostname: centos-kubernetes-02.localdomain
Capacity:
cpu: 4
ephemeral-storage: 17394Mi
hugepages-2Mi: 0
memory: 3873244Ki
pods: 110
Allocatable:
cpu: 4
ephemeral-storage: 16415037823
hugepages-2Mi: 0
memory: 3770844Ki
pods: 110
System Info:
Machine ID: 71fa38d22880454a88704a8c92aa7cac
System UUID: 564DE942-4986-5D27-8A09-E257D6BD7420
Boot ID: 87c70695-a57f-4631-acee-8b8b24985d2b
Kernel Version: 3.10.0-693.21.1.el7.x86_64
OS Image: CentOS Linux 7 (Core)
Operating System: linux
Architecture: amd64
Container Runtime Version: docker://1.13.1
Kubelet Version: v1.10.2
Kube-Proxy Version: v1.10.2
PodCIDR: 10.244.1.0/24
ExternalID: centos-kubernetes-02.localdomain
Non-terminated Pods: (3 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits
--------- ---- ------------ ---------- --------------- -------------
default signalfx-agent-kctbx 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system calico-node-scgq4 250m (6%) 0 (0%) 0 (0%) 0 (0%)
kube-system kube-proxy-hqw79 0 (0%) 0 (0%) 0 (0%) 0 (0%)
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
CPU Requests CPU Limits Memory Requests Memory Limits
------------ ---------- --------------- -------------
250m (6%) 0 (0%) 0 (0%) 0 (0%)
Events: <none>
Name: centos-kubernetes-03.localdomain
Roles: <none>
Labels: beta.kubernetes.io/arch=amd64
beta.kubernetes.io/os=linux
kubernetes.io/hostname=centos-kubernetes-03.localdomain
Annotations: node.alpha.kubernetes.io/ttl=0
volumes.kubernetes.io/controller-managed-attach-detach=true
CreationTimestamp: Fri, 27 Apr 2018 12:30:18 -0500
Taints: <none>
Unschedulable: false
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
OutOfDisk False Tue, 01 May 2018 18:53:01 -0500 Tue, 01 May 2018 17:12:04 -0500 KubeletHasSufficientDisk kubelet has sufficient disk space available
MemoryPressure False Tue, 01 May 2018 18:53:01 -0500 Tue, 01 May 2018 17:12:04 -0500 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Tue, 01 May 2018 18:53:01 -0500 Tue, 01 May 2018 17:12:04 -0500 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Tue, 01 May 2018 18:53:01 -0500 Fri, 27 Apr 2018 12:30:18 -0500 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Tue, 01 May 2018 18:53:01 -0500 Tue, 01 May 2018 17:12:14 -0500 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.1.117
Hostname: centos-kubernetes-03.localdomain
Capacity:
cpu: 4
ephemeral-storage: 17394Mi
hugepages-2Mi: 0
memory: 3873244Ki
pods: 110
Allocatable:
cpu: 4
ephemeral-storage: 16415037823
hugepages-2Mi: 0
memory: 3770844Ki
pods: 110
System Info:
Machine ID: 51bd98144dcc4558b55554f0e176dcab
System UUID: 564D7495-8C9E-5CD2-FEE2-D01675BB0485
Boot ID: 59794dc3-35b4-41df-9568-d0ea0041e150
Kernel Version: 3.10.0-693.21.1.el7.x86_64
OS Image: CentOS Linux 7 (Core)
Operating System: linux
Architecture: amd64
Container Runtime Version: docker://1.13.1
Kubelet Version: v1.10.2
Kube-Proxy Version: v1.10.2
PodCIDR: 10.244.2.0/24
ExternalID: centos-kubernetes-03.localdomain
Non-terminated Pods: (3 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits
--------- ---- ------------ ---------- --------------- -------------
default signalfx-agent-d9p55 0 (0%) 0 (0%) 0 (0%) 0 (0%)
kube-system calico-node-k4dqs 250m (6%) 0 (0%) 0 (0%) 0 (0%)
kube-system kube-proxy-gf7gq 0 (0%) 0 (0%) 0 (0%) 0 (0%)
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
CPU Requests CPU Limits Memory Requests Memory Limits
------------ ---------- --------------- -------------
250m (6%) 0 (0%) 0 (0%) 0 (0%)
Events: <none>
$ kubectl logs signalfx-agent-wrp7r
<...snip...>
time="2020-02-19T22:19:12Z" level=info msg="Watching for Docker events since 2020-02-19T22:19:12.147796497Z" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container c1bcdf42919b428db5a24dbd465a42ec9fc6cd093aca6c61ca9c4648ae85ce55" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container 292b504c28da27e27d90cc236b40622387f2ec84da7b133a18a58242fd80dd6d" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container d5845cf0c47c86fbf6e41080d2c67ad2042c313b02501d16f39b19f73514e772" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container 475c9c67f234229749d62485d0c971d2f00fec660781f37fbc189ab876d6f83d" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container 87c7433d4052c74ab008cf46570ab39c0bf4c14ee56aecff63539e904f8031c2" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container 2088fa9136ef677260483df8d1788a4401b1c58615abe210d050bcd170a26472" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container 4df94719e201c8d0d0a08fdc155a314c562e44d38dda860f99aa1e0d33225a44" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container b39785bbf99272bc587634bb056b50bf66007bdfda5201156d1ffd75ac479c6b" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"vonavi_http\") == \"80\"" monitorID=17 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"jenny\") == \"8675309\"" monitorID=18 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"app.vonavi.http\") == \"80\"" monitorID=19 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"vonavi_http\") == \"80\" && Get(container_labels, \"vonavi_https\") == ToString(port)" monitorID=20 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"vonavi_http\") == \"80\"" monitorID=21 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"jenny\") == \"8675309\"" monitorID=22 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"vonavi.http\") == \"80\" && Get(container_labels, \"vonavi.https\") == ToString(port)" monitorID=23 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Creating new monitor" discoveryRule="Get(container_labels, \"app.vonavi.http\") == \"80\"" monitorID=24 monitorType=collectd/nginx
time="2020-02-19T22:19:12Z" level=info msg="Monitoring docker container 226bd0d5b6fc77178fd9f1cc4726d620e7eec410a9b1b26496f078038c3bbf78" monitorType=docker-container-stats
time="2020-02-19T22:19:12Z" level=info msg="K8s leader is now node vonavi-k8s-1-12-1-do-3-nyc1-1-je04"
time="2020-02-19T22:19:12Z" level=info msg="Starting K8s API resource sync"
<...snip...>
time="2020-02-19T22:19:15Z" level=info msg="plugin \"nginx\" successfully loaded." collectdInstance=global plugin=plugin_load
<...snip...>
Run Smart Agent status commands $ kubectl exec -it signalfx-agent-wrp7r signalfx-agent status all
SignalFx Agent version: 4.7.1
Agent uptime: 1m2s
Observers active: k8s-api
Active Monitors: 16
Configured Monitors: 39
Discovered Endpoint Count: 1
Bad Monitor Config: None
Global Dimensions: {kubernetes_cluster: vonavi-do, host: vonavi-k8s-1-12-1-do-3-nyc1-1-38ast.localdomain, kubernetes_node: vonavi-k8s-1-12-1-do-3-nyc1-1-38ast}
Datapoints sent (last minute): 1832
Events Sent (last minute): 18
Trace Spans Sent (last minute): 0
Kubernetes Leader Node: vonavi-k8s-1-12-1-do-3-nyc1-1-je04
Agent Configuration:
signalFxAccessToken: ***************
ingestUrl: https://ingest.us0.signalfx.com
traceEndpointUrl: https://ingest.us0.signalfx.com/v1/trace
apiUrl: https://api.us0.signalfx.com
signalFxRealm: us0
hostname: ""
useFullyQualifiedHost:
disableHostDimensions: false
intervalSeconds: 10
globalDimensions:
kubernetes_cluster: vonavi-do
sendMachineID: false
validateDiscoveryRules: true
observers:
monitors:
- type: collectd/cpu
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/cpufreq
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/df
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/disk
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/interface
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/load
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/memory
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/protocols
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/signalfx-metadata
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: host-metadata
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/uptime
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/vmem
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: collectd/processes
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: kubelet-stats
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: kubernetes-cluster
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
- type: docker-container-stats
discoveryRule: ""
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
<...snip...>
24. collectd/nginx
Reporting Interval (seconds): 10
Enabled Metrics: [
connections.handled, nginx_connections.active, nginx_connections.reading, nginx_connections.waiting,
nginx_connections.writing, nginx_requests, connections.accepted
]
Discovery Rule: Get(container_labels, "app.vonavi.http") == "80"
Monitored Endpoint ID: nginx-ingress-controller-5bcc95cbb-nsjlf-ff567a4-443 Config:
type: collectd/nginx
discoveryRule: Get(container_labels, "app.vonavi.http") == "80"
validateDiscoveryRule: true
extraDimensions:
extraDimensionsFromEndpoint:
configEndpointMappings:
intervalSeconds: 10
solo: false
metricsToExclude:
datapointsToExclude:
disableHostDimensions: false
disableEndpointDimensions: false
dimensionTransformations:
extraMetrics:
extraGroups:
host: 10.244.24.2
port: 443
name: https
url: http://{{.Host}}:{{.Port}}/nginx_status
username: ""
password:
timeout: 0
Delete the signalfx-agent service account ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl delete serviceaccounts/signalfx-agent
serviceaccount "signalfx-agent" deleted
Delete the signalfx-agent daemonset ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl delete ds signalfx-agent --force --now
daemonset.extensions "signalfx-agent" deleted
Delete the signalfx-agent role binding ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl delete clusterrolebinding/signalfx-agent
clusterrolebinding.rbac.authorization.k8s.io "signalfx-agent" deleted
Delete the signalfx-agent cluster role ╭─jivanov@centos-kubernetes-01 ~
╰─$ kubectl delete clusterrole/signalfx-agent
clusterrole.rbac.authorization.k8s.io "signalfx-agent" deleted
