kube-proxy hangs - goroutines

kube-proxy-goroutines.txt

(dlv) goroutines
[79 goroutines]
  Goroutine 1 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/iptables/proxier.go:421 k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).SyncLoop (0xdd5f7e)
  Goroutine 2 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 3 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 4 - User: .usr/local/go/src/runtime/proc.go:2524 runtime.exitsyscall (0x4394d4)
  Goroutine 17 - User: .usr/local/go/src/runtime/asm_amd64.s:2087 runtime.goexit (0x464081)
  Goroutine 18 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 19 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/golang/glog/glog.go:879 k8s.io/kubernetes/vendor/github.com/golang/glog.(*loggingT).flushDaemon (0x56a331)
  Goroutine 20 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 21 - User: .usr/local/go/src/runtime/mbitmap.go:388 runtime.heapBitsForObject (0x417350)
  Goroutine 22 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 23 - User: .usr/local/go/src/runtime/mgcmark.go:1120 runtime.scanobject (0x422330)
  Goroutine 24 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 25 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 26 - User: .usr/local/go/src/runtime/mbitmap.go:388 runtime.heapBitsForObject (0x417350)
  Goroutine 27 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 28 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 29 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 30 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 31 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 32 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 33 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 34 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 35 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 36 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 37 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 38 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 39 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 40 - User: .usr/local/go/src/runtime/asm_amd64.s:252 runtime.systemstack_switch (0x461130)
  Goroutine 41 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 42 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 43 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 44 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 45 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 46 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 47 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 48 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 49 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 50 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 51 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 52 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 53 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 54 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 55 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 56 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 57 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 58 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 59 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 60 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 61 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 62 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 63 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 64 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 65 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 66 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 67 - User: .usr/local/go/src/runtime/proc.go:260 runtime.gopark (0x43307a)
  Goroutine 90 - User: .usr/local/go/src/runtime/sigqueue.go:116 os/signal.signal_recv (0x44669e)
  Goroutine 93 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/iptables/iptables.go:530 k8s.io/kubernetes/pkg/util/iptables.(*runner).dbusSignalHandler (0xe04997)
  Goroutine 94 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/watch/mux.go:201 k8s.io/kubernetes/pkg/watch.(*Broadcaster).loop (0xfbf331)
  Goroutine 131 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:108 k8s.io/kubernetes/pkg/util/wait.JitterUntil (0x88fada)
  Goroutine 133 - User: .usr/local/go/src/runtime/netpoll.go:160 net.runtime_pollWait (0x42d0b2)
  Goroutine 134 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/godbus/dbus/conn.go:385 k8s.io/kubernetes/vendor/github.com/godbus/dbus.(*Conn).outWorker (0x2053011)
  Goroutine 135 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/healthcheck/worker.go:33 k8s.io/kubernetes/pkg/proxy/healthcheck.(*proxyHC).handlerLoop (0x204b80d)
  Goroutine 144 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/config/config.go:296 k8s.io/kubernetes/pkg/proxy/config.watchForUpdates (0xdd294d)
* Goroutine 145 - User: .usr/local/go/src/syscall/asm_linux_amd64.s:53 syscall.Syscall6 (0xa7c67a) (thread 24)
  Goroutine 149 - User: .usr/local/go/src/runtime/netpoll.go:160 net.runtime_pollWait (0x42d0b2)
  Goroutine 178 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/config/config.go:82 k8s.io/kubernetes/pkg/util/config.(*Mux).listen (0xd56bc1)
  Goroutine 179 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/config/config.go:217 k8s.io/kubernetes/pkg/proxy/config.(*ServiceConfig).Channel.func1 (0xdd3274)
  Goroutine 180 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/config/config.go:82 k8s.io/kubernetes/pkg/util/config.(*Mux).listen (0xd56bc1)
  Goroutine 181 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/config/config.go:108 k8s.io/kubernetes/pkg/proxy/config.(*EndpointsConfig).Channel.func1 (0xdd3004)
  Goroutine 182 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:346 k8s.io/kubernetes/pkg/client/cache.(*Reflector).watchHandler (0x1706c37)
  Goroutine 183 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:346 k8s.io/kubernetes/pkg/client/cache.(*Reflector).watchHandler (0x1706c37)
  Goroutine 184 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/record/event.go:228 k8s.io/kubernetes/pkg/client/record.(*eventBroadcasterImpl).StartEventWatcher.func1 (0xbefedc)
  Goroutine 185 - User: .usr/local/go/src/runtime/netpoll.go:160 net.runtime_pollWait (0x42d0b2)
  Goroutine 188 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:266 k8s.io/kubernetes/pkg/client/cache.(*Reflector).ListAndWatch.func1 (0x1710eaa)
  Goroutine 189 - User: .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:266 k8s.io/kubernetes/pkg/client/cache.(*Reflector).ListAndWatch.func1 (0x1710eaa)
  Goroutine 308820 - User: .usr/local/go/src/runtime/sema.go:267 sync.runtime_notifyListWait (0x443951)
  Goroutine 310190 - User: .usr/local/go/src/runtime/sema.go:267 sync.runtime_notifyListWait (0x443951)
  Goroutine 313922 - User: .usr/local/go/src/syscall/asm_linux_amd64.s:39 syscall.Syscall (0xa7c63f)
  Goroutine 313923 - User: .usr/local/go/src/syscall/asm_linux_amd64.s:27 syscall.Syscall (0xa7c604) (thread 33)
(dlv) goroutine 1 bt
 0  0x000000000043307a in runtime.gopark
    at .usr/local/go/src/runtime/proc.go:260
 1  0x000000000043312d in runtime.goparkunlock
    at .usr/local/go/src/runtime/proc.go:265
 2  0x000000000040aa9e in runtime.chanrecv
    at .usr/local/go/src/runtime/chan.go:496
 3  0x000000000040a76e in runtime.chanrecv1
    at .usr/local/go/src/runtime/chan.go:378
 4  0x0000000000dd5f7e in k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).SyncLoop
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/iptables/proxier.go:421
 5  0x00000000004beadc in k8s.io/kubernetes/cmd/kube-proxy/app.(*ProxyServer).Run
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-proxy/app/server.go:376
 6  0x000000000040616e in main.NewKubeProxy.func1
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/hyperkube/kube-proxy.go:50
 7  0x0000000000403812 in main.(*HyperKube).Run
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/hyperkube/hyperkube.go:179
 8  0x0000000000403e99 in main.(*HyperKube).RunToExit
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/hyperkube/hyperkube.go:189
 9  0x0000000000000000 in runtime.morestack_noctxt
    at :-1
10  0x000000c42038e060 in (nil)
    at ?:-1
(dlv) goroutine 2 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x0000000000432e4c in runtime.forcegchelper
   at .usr/local/go/src/runtime/proc.go:224
3  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 3 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x0000000000423332 in runtime.bgsweep
   at .usr/local/go/src/runtime/mgcsweep.go:79
3  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 4 bt
0  0x00000000004394d4 in runtime.exitsyscall
   at .usr/local/go/src/runtime/proc.go:2524
1  0x0000000000414693 in runtime.notetsleepg
   at .usr/local/go/src/runtime/lock_futex.go:207
2  0x000000000044eac8 in runtime.timerproc
   at .usr/local/go/src/runtime/time.go:209
3  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 17 bt
0  0x0000000000464081 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2087
(dlv) goroutine 18 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000041b0a3 in runtime.runfinq
   at .usr/local/go/src/runtime/mfinal.go:158
3  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 19 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x000000000056a331 in k8s.io/kubernetes/vendor/github.com/golang/glog.(*loggingT).flushDaemon
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/golang/glog/glog.go:879
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 20 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 21 bt
Command failed: NULL address
(dlv) goroutine 22 bt
Command failed: input/output error
(dlv) goroutine 23 bt
Command failed: NULL address
(dlv) goroutine 24 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 25 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 26 bt
Command failed: NULL address
(dlv) goroutine 27 bt
Command failed: input/output error
(dlv) goroutine 28 bt
Command failed: input/output error
(dlv) goroutine 29 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 30 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 31 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 32 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 33 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 34 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 35 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 36 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 37 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 38 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 39 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 40 bt
0  0x0000000000461130 in runtime.systemstack_switch
   at .usr/local/go/src/runtime/asm_amd64.s:252
1  0x000000000041d825 in runtime.gcMarkTermination
   at .usr/local/go/src/runtime/mgc.go:1186
2  0x000000000041d6a9 in runtime.gcMarkDone
   at .usr/local/go/src/runtime/mgc.go:1150
3  0x000000000041e61d in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1497
4  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 41 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 42 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 43 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 44 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 45 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 46 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 47 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 48 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 49 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 50 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 51 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 52 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 53 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 54 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 55 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 56 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 57 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 58 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 59 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 60 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 61 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 62 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 63 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 64 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 65 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 66 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 67 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000041e42b in runtime.gcBgMarkWorker
   at .usr/local/go/src/runtime/mgc.go:1420
2  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 90 bt
0  0x0000000000414672 in runtime.notetsleepg
   at .usr/local/go/src/runtime/lock_futex.go:206
1  0x000000000044669e in os/signal.signal_recv
   at .usr/local/go/src/runtime/sigqueue.go:116
2  0x0000000000fc677d in os/signal.loop
   at .usr/local/go/src/os/signal/signal_unix.go:22
3  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 93 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000e04997 in k8s.io/kubernetes/pkg/util/iptables.(*runner).dbusSignalHandler
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/iptables/iptables.go:530
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 94 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000fbf331 in k8s.io/kubernetes/pkg/watch.(*Broadcaster).loop
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/watch/mux.go:201
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 131 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000044273d in runtime.selectgoImpl
   at .usr/local/go/src/runtime/select.go:423
2  0x000000000044155e in runtime.selectgo
   at .usr/local/go/src/runtime/select.go:238
3  0x000000000088fada in k8s.io/kubernetes/pkg/util/wait.JitterUntil
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:108
4  0x000000000088f96d in k8s.io/kubernetes/pkg/util/wait.Until
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:52
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 133 bt
 0  0x000000000043307a in runtime.gopark
    at .usr/local/go/src/runtime/proc.go:260
 1  0x000000000042d92e in runtime.netpollblock
    at .usr/local/go/src/runtime/netpoll.go:341
 2  0x000000000042d0b2 in net.runtime_pollWait
    at .usr/local/go/src/runtime/netpoll.go:160
 3  0x00000000008a763d in net.(*pollDesc).wait
    at .usr/local/go/src/net/fd_poll_runtime.go:73
 4  0x00000000008a76ad in net.(*pollDesc).waitRead
    at .usr/local/go/src/net/fd_poll_runtime.go:78
 5  0x00000000008a97c2 in net.(*netFD).readMsg
    at .usr/local/go/src/net/fd_unix.go:297
 6  0x00000000008cde3f in net.(*UnixConn).readMsg
    at .usr/local/go/src/net/unixsock_posix.go:111
 7  0x00000000008cad53 in net.(*UnixConn).ReadMsgUnix
    at .usr/local/go/src/net/unixsock.go:127
 8  0x00000000020648ff in k8s.io/kubernetes/vendor/github.com/godbus/dbus.(*oobReader).Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/godbus/dbus/transport_unix.go:21
 9  0x0000000000484c9a in io.ReadAtLeast
    at .usr/local/go/src/io/io.go:307
10  0x0000000000000000 in runtime.morestack_noctxt
    at :-1
(dlv) goroutine 134 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000002053011 in k8s.io/kubernetes/vendor/github.com/godbus/dbus.(*Conn).outWorker
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/godbus/dbus/conn.go:385
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 135 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000044273d in runtime.selectgoImpl
   at .usr/local/go/src/runtime/select.go:423
2  0x000000000044155e in runtime.selectgo
   at .usr/local/go/src/runtime/select.go:238
3  0x000000000204b80d in k8s.io/kubernetes/pkg/proxy/healthcheck.(*proxyHC).handlerLoop
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/healthcheck/worker.go:33
4  0x000000000204bded in k8s.io/kubernetes/pkg/proxy/healthcheck.Run.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/healthcheck/api.go:63
5  0x0000000000890324 in k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:96
6  0x000000000088faac in k8s.io/kubernetes/pkg/util/wait.JitterUntil
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:97
7  0x0000000000000000 in runtime.morestack_noctxt
   at :-1
8  0x000000000435aa90 in (nil)
   at ?:-1
(dlv) goroutine 144 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a76e in runtime.chanrecv1
   at .usr/local/go/src/runtime/chan.go:378
4  0x0000000000dd294d in k8s.io/kubernetes/pkg/proxy/config.watchForUpdates
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/config/config.go:296
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 145 bt
 0  0x0000000000a7c67a in syscall.Syscall6
    at .usr/local/go/src/syscall/asm_linux_amd64.s:53
 1  0x0000000000491704 in os.(*Process).blockUntilWaitable
    at .usr/local/go/src/os/wait_waitid.go:28
 2  0x000000000048b172 in os.(*Process).wait
    at .usr/local/go/src/os/exec_unix.go:22
 3  0x000000000048988d in os.(*Process).Wait
    at .usr/local/go/src/os/doc.go:49
 4  0x0000000001d5846d in os/exec.(*Cmd).Wait
    at .usr/local/go/src/os/exec/exec.go:434
 5  0x0000000001d5788a in os/exec.(*Cmd).Run
    at .usr/local/go/src/os/exec/exec.go:279
 6  0x0000000001d58ca7 in os/exec.(*Cmd).CombinedOutput
    at .usr/local/go/src/os/exec/exec.go:494
 7  0x0000000000dfecc1 in k8s.io/kubernetes/pkg/util/exec.(*cmdWrapper).CombinedOutput
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/exec/exec.go:98
 8  0x0000000000e023f0 in k8s.io/kubernetes/pkg/util/iptables.(*runner).restoreInternal
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/iptables/iptables.go:342
 9  0x0000000000e02072 in k8s.io/kubernetes/pkg/util/iptables.(*runner).RestoreAll
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/iptables/iptables.go:323
10  0x0000000000ddd9d6 in k8s.io/kubernetes/pkg/proxy/iptables.(*Proxier).syncProxyRules
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/iptables/proxier.go:1312
(dlv) goroutine 149 bt
 0  0x000000000043307a in runtime.gopark
    at .usr/local/go/src/runtime/proc.go:260
 1  0x000000000042d92e in runtime.netpollblock
    at .usr/local/go/src/runtime/netpoll.go:341
 2  0x000000000042d0b2 in net.runtime_pollWait
    at .usr/local/go/src/runtime/netpoll.go:160
 3  0x00000000008a763d in net.(*pollDesc).wait
    at .usr/local/go/src/net/fd_poll_runtime.go:73
 4  0x00000000008a76ad in net.(*pollDesc).waitRead
    at .usr/local/go/src/net/fd_poll_runtime.go:78
 5  0x00000000008a8ef3 in net.(*netFD).Read
    at .usr/local/go/src/net/fd_unix.go:243
 6  0x00000000008bb5d0 in net.(*conn).Read
    at .usr/local/go/src/net/net.go:173
 7  0x0000000000ada2a6 in crypto/tls.(*block).readFromUntil
    at .usr/local/go/src/crypto/tls/conn.go:476
 8  0x0000000000ada7e5 in crypto/tls.(*Conn).readRecord
    at .usr/local/go/src/crypto/tls/conn.go:578
 9  0x0000000000000000 in runtime.morestack_noctxt
    at :-1
10  0x000000c4208d6380 in (nil)
    at ?:-1
(dlv) goroutine 178 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000d56bc1 in k8s.io/kubernetes/pkg/util/config.(*Mux).listen
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/config/config.go:82
5  0x0000000000d569df in k8s.io/kubernetes/pkg/util/config.(*Mux).Channel.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/config/config.go:77
6  0x0000000000890324 in k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:96
7  0x000000000088faac in k8s.io/kubernetes/pkg/util/wait.JitterUntil
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:97
8  0x000000000088f96d in k8s.io/kubernetes/pkg/util/wait.Until
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:52
9  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 179 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000dd3274 in k8s.io/kubernetes/pkg/proxy/config.(*ServiceConfig).Channel.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/config/config.go:217
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 180 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000d56bc1 in k8s.io/kubernetes/pkg/util/config.(*Mux).listen
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/config/config.go:82
5  0x0000000000d569df in k8s.io/kubernetes/pkg/util/config.(*Mux).Channel.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/config/config.go:77
6  0x0000000000890324 in k8s.io/kubernetes/pkg/util/wait.JitterUntil.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:96
7  0x000000000088faac in k8s.io/kubernetes/pkg/util/wait.JitterUntil
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:97
8  0x000000000088f96d in k8s.io/kubernetes/pkg/util/wait.Until
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/wait/wait.go:52
9  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 181 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000dd3004 in k8s.io/kubernetes/pkg/proxy/config.(*EndpointsConfig).Channel.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/proxy/config/config.go:108
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 182 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000044273d in runtime.selectgoImpl
   at .usr/local/go/src/runtime/select.go:423
2  0x000000000044155e in runtime.selectgo
   at .usr/local/go/src/runtime/select.go:238
3  0x0000000001706c37 in k8s.io/kubernetes/pkg/client/cache.(*Reflector).watchHandler
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:346
4  0x0000000000000000 in runtime.morestack_noctxt
   at :-1
5  0x000000c420945c80 in (nil)
   at ?:-1
(dlv) goroutine 183 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000044273d in runtime.selectgoImpl
   at .usr/local/go/src/runtime/select.go:423
2  0x000000000044155e in runtime.selectgo
   at .usr/local/go/src/runtime/select.go:238
3  0x0000000001706c37 in k8s.io/kubernetes/pkg/client/cache.(*Reflector).watchHandler
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:346
4  0x0000000000000000 in runtime.morestack_noctxt
   at :-1
5  0x000000c420945d00 in (nil)
   at ?:-1
(dlv) goroutine 184 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000043312d in runtime.goparkunlock
   at .usr/local/go/src/runtime/proc.go:265
2  0x000000000040aa9e in runtime.chanrecv
   at .usr/local/go/src/runtime/chan.go:496
3  0x000000000040a7ae in runtime.chanrecv2
   at .usr/local/go/src/runtime/chan.go:383
4  0x0000000000befedc in k8s.io/kubernetes/pkg/client/record.(*eventBroadcasterImpl).StartEventWatcher.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/record/event.go:228
5  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 185 bt
 0  0x000000000043307a in runtime.gopark
    at .usr/local/go/src/runtime/proc.go:260
 1  0x000000000042d92e in runtime.netpollblock
    at .usr/local/go/src/runtime/netpoll.go:341
 2  0x000000000042d0b2 in net.runtime_pollWait
    at .usr/local/go/src/runtime/netpoll.go:160
 3  0x00000000008a763d in net.(*pollDesc).wait
    at .usr/local/go/src/net/fd_poll_runtime.go:73
 4  0x00000000008a76ad in net.(*pollDesc).waitRead
    at .usr/local/go/src/net/fd_poll_runtime.go:78
 5  0x00000000008aa807 in net.(*netFD).accept
    at .usr/local/go/src/net/fd_unix.go:419
 6  0x00000000008c634d in net.(*TCPListener).accept
    at .usr/local/go/src/net/tcpsock_posix.go:132
 7  0x00000000008c4880 in net.(*TCPListener).AcceptTCP
    at .usr/local/go/src/net/tcpsock.go:209
 8  0x00000000009247e1 in net/http.tcpKeepAliveListener.Accept
    at .usr/local/go/src/net/http/server.go:2608
 9  0x00000000009239f4 in net/http.(*Server).Serve
    at .usr/local/go/src/net/http/server.go:2273
10  0x0000000000923710 in net/http.(*Server).ListenAndServe
    at .usr/local/go/src/net/http/server.go:2219
(dlv) goroutine 188 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000044273d in runtime.selectgoImpl
   at .usr/local/go/src/runtime/select.go:423
2  0x000000000044155e in runtime.selectgo
   at .usr/local/go/src/runtime/select.go:238
3  0x0000000001710eaa in k8s.io/kubernetes/pkg/client/cache.(*Reflector).ListAndWatch.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:266
4  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 189 bt
0  0x000000000043307a in runtime.gopark
   at .usr/local/go/src/runtime/proc.go:260
1  0x000000000044273d in runtime.selectgoImpl
   at .usr/local/go/src/runtime/select.go:423
2  0x000000000044155e in runtime.selectgo
   at .usr/local/go/src/runtime/select.go:238
3  0x0000000001710eaa in k8s.io/kubernetes/pkg/client/cache.(*Reflector).ListAndWatch.func1
   at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/client/cache/reflector.go:266
4  0x0000000000464080 in runtime.goexit
   at .usr/local/go/src/runtime/asm_amd64.s:2086
(dlv) goroutine 308820 bt
 0  0x000000000043307a in runtime.gopark
    at .usr/local/go/src/runtime/proc.go:260
 1  0x000000000043312d in runtime.goparkunlock
    at .usr/local/go/src/runtime/proc.go:265
 2  0x0000000000443951 in sync.runtime_notifyListWait
    at .usr/local/go/src/runtime/sema.go:267
 3  0x0000000000a6143a in sync.(*Cond).Wait
    at .usr/local/go/src/sync/cond.go:57
 4  0x0000000001af052c in k8s.io/kubernetes/vendor/golang.org/x/net/http2.(*pipe).Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/golang.org/x/net/http2/pipe.go:60
 5  0x0000000001b067b0 in k8s.io/kubernetes/vendor/golang.org/x/net/http2.transportResponseBody.Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/golang.org/x/net/http2/transport.go:1465
 6  0x0000000000484c9a in io.ReadAtLeast
    at .usr/local/go/src/io/io.go:307
 7  0x00000000020f4a99 in k8s.io/kubernetes/pkg/util/framer.(*lengthDelimitedFrameReader).Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/framer/framer.go:76
 8  0x00000000017dcb71 in k8s.io/kubernetes/pkg/runtime/serializer/streaming.(*decoder).Decode
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/runtime/serializer/streaming/streaming.go:77
 9  0x00000000014964ce in k8s.io/kubernetes/pkg/watch/versioned.(*Decoder).Decode
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/watch/versioned/decoder.go:48
10  0x0000000000000000 in runtime.morestack_noctxt
    at :-1
(dlv) goroutine 310190 bt
 0  0x000000000043307a in runtime.gopark
    at .usr/local/go/src/runtime/proc.go:260
 1  0x000000000043312d in runtime.goparkunlock
    at .usr/local/go/src/runtime/proc.go:265
 2  0x0000000000443951 in sync.runtime_notifyListWait
    at .usr/local/go/src/runtime/sema.go:267
 3  0x0000000000a6143a in sync.(*Cond).Wait
    at .usr/local/go/src/sync/cond.go:57
 4  0x0000000001af052c in k8s.io/kubernetes/vendor/golang.org/x/net/http2.(*pipe).Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/golang.org/x/net/http2/pipe.go:60
 5  0x0000000001b067b0 in k8s.io/kubernetes/vendor/golang.org/x/net/http2.transportResponseBody.Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/golang.org/x/net/http2/transport.go:1465
 6  0x0000000000484c9a in io.ReadAtLeast
    at .usr/local/go/src/io/io.go:307
 7  0x00000000020f4a99 in k8s.io/kubernetes/pkg/util/framer.(*lengthDelimitedFrameReader).Read
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/util/framer/framer.go:76
 8  0x00000000017dcb71 in k8s.io/kubernetes/pkg/runtime/serializer/streaming.(*decoder).Decode
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/runtime/serializer/streaming/streaming.go:77
 9  0x00000000014964ce in k8s.io/kubernetes/pkg/watch/versioned.(*Decoder).Decode
    at .go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/pkg/watch/versioned/decoder.go:48
10  0x0000000000000000 in runtime.morestack_noctxt
    at :-1
(dlv) goroutine 313922 bt
 0  0x00000000004394d4 in runtime.exitsyscall
    at .usr/local/go/src/runtime/proc.go:2524
 1  0x0000000000a7c63f in syscall.Syscall
    at .usr/local/go/src/syscall/asm_linux_amd64.s:39
 2  0x0000000000a78a24 in syscall.write
    at .usr/local/go/src/syscall/zsyscall_linux_amd64.go:1064
 3  0x0000000000a758dd in syscall.Write
    at .usr/local/go/src/syscall/syscall_unix.go:180
 4  0x000000000048ea52 in os.(*File).write
    at .usr/local/go/src/os/file_unix.go:249
 5  0x000000000048be8f in os.(*File).Write
    at .usr/local/go/src/os/file.go:142
 6  0x0000000000a64f54 in bytes.(*Buffer).WriteTo
    at .usr/local/go/src/bytes/buffer.go:209
 7  0x00000000004854f7 in io.copyBuffer
    at .usr/local/go/src/io/io.go:380
 8  0x0000000000484fe1 in io.Copy
    at .usr/local/go/src/io/io.go:360
 9  0x0000000001d5ab5e in os/exec.(*Cmd).stdin.func1
    at .usr/local/go/src/os/exec/exec.go:208
10  0x0000000001d5aced in os/exec.(*Cmd).Start.func1
    at .usr/local/go/src/os/exec/exec.go:370
(dlv) goroutine 313923 bt
 0  0x0000000000a7c604 in syscall.Syscall
    at .usr/local/go/src/syscall/asm_linux_amd64.s:27
 1  0x0000000000a78544 in syscall.read
    at .usr/local/go/src/syscall/zsyscall_linux_amd64.go:783
 2  0x0000000000a7585d in syscall.Read
    at .usr/local/go/src/syscall/syscall_unix.go:161
 3  0x000000000048e8ed in os.(*File).read
    at .usr/local/go/src/os/file_unix.go:228
 4  0x000000000048ba0f in os.(*File).Read
    at .usr/local/go/src/os/file.go:101
 5  0x0000000000a64c5c in bytes.(*Buffer).ReadFrom
    at .usr/local/go/src/bytes/buffer.go:176
 6  0x000000000048549f in io.copyBuffer
    at .usr/local/go/src/io/io.go:384
 7  0x0000000000484fe1 in io.Copy
    at .usr/local/go/src/io/io.go:360
 8  0x0000000001d5ac5e in os/exec.(*Cmd).writerDescriptor.func1
    at .usr/local/go/src/os/exec/exec.go:253
 9  0x0000000001d5aced in os/exec.(*Cmd).Start.func1
    at .usr/local/go/src/os/exec/exec.go:370
10  0x0000000000464080 in runtime.goexit
    at .usr/local/go/src/runtime/asm_amd64.s:2086