Stolon sentinel locking up due to etcd

11

λ (lab) ~> kl stolon-main-sentinel-backend-stolon-main-sentinel-78d4d597c2lsv -f
2019-05-16T02:57:57.750Z        INFO    cmd/sentinel.go:1962    sentinel uid    {"uid": "f3e6c1b3"}
2019-05-16T02:57:57.850Z        INFO    cmd/sentinel.go:80      Trying to acquire sentinels leadership
2019-05-16T07:19:41.806Z        ERROR   cmd/sentinel.go:100     election loop error     {"error": "lost watcher waiting for delete"}
2019-05-16T07:19:51.806Z        INFO    cmd/sentinel.go:80      Trying to acquire sentinels leadership
2019-05-16T07:25:43.371Z        ERROR   cmd/sentinel.go:1815    error retrieving cluster data   {"error": "context deadline exceeded"}


SIGQUIT: quit
PC=0x45ae01 m=0 sigcode=0

goroutine 6 [select, 73 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateNotifyLoop(0xc0004ea100)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:349 +0x33f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:128 +0x2ed

goroutine 7 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateUnhealthy(0xc0004ea100)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:246 +0xcf
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer.func1(0xc0004ea100)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:132 +0x5a
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:130 +0x32d

goroutine 8 [select, 73 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0003be4c0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x110
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newCCBalancerWrapper
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x14f

goroutine 9 [chan receive, 73 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapper).lbWatcher(0xc000122000)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:122 +0xb6
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapperBuilder).Build
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:60 +0x29d

goroutine 11 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateNotifyLoop(0xc0004ea200)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:349 +0x33f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:128 +0x2ed

goroutine 12 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*healthBalancer).updateUnhealthy(0xc0004ea200)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:246 +0xcf
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer.func1(0xc0004ea200)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:132 +0x5a
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.newHealthBalancer
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/health_balancer.go:130 +0x32d

goroutine 13 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0004ce600)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x110
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newCCBalancerWrapper
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x14f

goroutine 14 [chan receive, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapper).lbWatcher(0xc000122070)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:122 +0xb6
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*balancerWrapperBuilder).Build
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/balancer_v1_wrapper.go:60 +0x29d

goroutine 31910 [chan receive, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc000203180)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:1067 +0x9ec
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).connect
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:684 +0xb6

// Primary start loop. Just waiting on our timer channel and the ctx.Done
goroutine 16 [select]:
github.com/sorintlab/stolon/cmd/sentinel/cmd.(*Sentinel).Start(0xc00032e300, 0x188a4c0, 0xc0003bfec0)
        /go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:1786 +0x16c
created by github.com/sorintlab/stolon/cmd/sentinel/cmd.sentinel
        /go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:1985 +0x3fa

// This is the election loop. We're paused for 79m waiting on something from
// either the electionCh, errCh or ctx.Done().
goroutine 82 [select, 79 minutes]:
github.com/sorintlab/stolon/cmd/sentinel/cmd.(*Sentinel).electionLoop(0xc00032e300, 0x188a4c0, 0xc0003bfec0)
        /go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:83 +0x130
created by github.com/sorintlab/stolon/cmd/sentinel/cmd.(*Sentinel).Start
        /go/src/github.com/sorintlab/stolon/cmd/sentinel/cmd/sentinel.go:1783 +0xab

goroutine 31933 [select, 79 minutes]:
context.propagateCancel.func1(0x188a7c0, 0xc0002e7d90, 0x18696c0, 0xc0001a1780)
        /usr/local/go/src/context/context.go:259 +0xd8
created by context.propagateCancel
        /usr/local/go/src/context/context.go:258 +0x18a

goroutine 99 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37bc8, 0x72, 0xffffffffffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000284918, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000284900, 0xc000310000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000284900, 0xc000310000, 0x1000, 0x1000, 0xc0003bcf80, 0xc0005eb938, 0x6cb90a)
        /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e2a8, 0xc000310000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x69
net/http.(*connReader).Read(0xc00027c030, 0xc000310000, 0x1000, 0x1000, 0x0, 0xc0005ebc00, 0x1dc0006cf26c)
        /usr/local/go/src/net/http/server.go:787 +0x107
bufio.(*Reader).fill(0xc000152180)
        /usr/local/go/src/bufio/bufio.go:100 +0x10f
bufio.(*Reader).ReadSlice(0xc000152180, 0x434c0a, 0x7ff573b07a90, 0xc0005ebb00, 0x40b779, 0xc0005f6100, 0x100)
        /usr/local/go/src/bufio/bufio.go:356 +0x3d
bufio.(*Reader).ReadLine(0xc000152180, 0xc0005ebb08, 0xc00006f500, 0x7ff575fa4008, 0x0, 0xc0002b0c90, 0x0)
        /usr/local/go/src/bufio/bufio.go:385 +0x34
net/textproto.(*Reader).readLineSlice(0xc0002b0c90, 0xc0005f6100, 0xc000284900, 0x0, 0x0, 0x42be11)
        /usr/local/go/src/net/textproto/reader.go:55 +0x6f
net/textproto.(*Reader).ReadLine(...)
        /usr/local/go/src/net/textproto/reader.go:36
net/http.readRequest(0xc000152180, 0x0, 0xc0005f6100, 0x0, 0x0)
        /usr/local/go/src/net/http/request.go:968 +0x8d
net/http.(*conn).readRequest(0xc0001a3c20, 0x188a4c0, 0xc000508580, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:967 +0x163
net/http.(*conn).serve(0xc0001a3c20, 0x188a4c0, 0xc000508580)
        /usr/local/go/src/net/http/server.go:1819 +0x6ad
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:2884 +0x2f4

goroutine 86 [select, 340 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).keepAliveCtxCloser(0xc000284700, 0x22596abc2e468ecf, 0x188a4c0, 0xc000508480, 0xc0006b6000)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:279 +0x10f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:246 +0x1d1

goroutine 87 [select]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0002520a0, 0xc0000a82b0, 0x5, 0x5, 0xc000044420, 0x20, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:158 +0xe5
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0002520a0, 0xc0000a82b0, 0x5, 0x5, 0x819d10, 0xc00036c000, 0xc0005359e0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:138 +0x127
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0000a1200, 0xc0000a82b0, 0x5, 0x5, 0xc000044420, 0xc000535a08, 0x82abfc)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:436 +0x55
io.ReadAtLeast(0x18591c0, 0xc0000a1200, 0xc0000a82b0, 0x5, 0x5, 0x5, 0x4e05dc, 0x1f, 0x2)
        /usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
        /usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000134000, 0xc0000a82b0, 0x5, 0x5, 0x155dc00, 0x7ff573d05388, 0xc000132120)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:420 +0xc8
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc0000a82a0, 0x400000, 0x2b, 0x2b, 0x0, 0x0, 0x1875840, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:508 +0x63
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recvAndDecompress(0xc0000a82a0, 0xc000134000, 0x0, 0x0, 0x400000, 0x0, 0x0, 0x0, 0xc0000ae120, 0x2b, ...)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:639 +0x4d
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recv(0xc0000a82a0, 0x7ff573a330c8, 0x25a7c48, 0xc000134000, 0x0, 0x0, 0x155dc00, 0xc0002b8640, 0x400000, 0x0, ...)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:684 +0x9b
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc00071c420, 0x155dc00, 0xc0002b8640, 0x0, 0x4054fc, 0xc0002b8640)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:861 +0xf0
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00071c420, 0xc000535e18, 0x203000)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:712 +0x46
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc000232ea0, 0xc000535e68, 0xc000535e58, 0x42be11, 0x8)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:564 +0x29f
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc000232ea0, 0x155dc00, 0xc0002b8640, 0xc00000e038, 0x2587560)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:711 +0x100
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*leaseLeaseKeepAliveClient).Recv(0xc0003b0020, 0xc0005a62e0, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:3346 +0x62
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).recvKeepAliveLoop(0xc000284700, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:396 +0x9e
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive.func1
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:248 +0x43

goroutine 88 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).deadlineLoop(0xc000284700)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:482 +0x18d
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive.func1
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:249 +0x65

goroutine 89 [chan receive]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession.func1(0xc0006b6300, 0xc0000c05a0)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:67 +0x73
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:65 +0x254

goroutine 31248 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream.func5(0xc000446300, 0xc000232ea0, 0x188a580, 0xc00028c1b0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:315 +0xd7
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:314 +0x943

goroutine 31867 [chan receive]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession.func1(0xc0005f84e0, 0xc0006a2600)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:67 +0x73
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.NewSession
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/session.go:65 +0x254

goroutine 31856 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37af8, 0x72, 0xffffffffffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00012a798, 0x72, 0x800, 0x846, 0xffffffffffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00012a780, 0xc00026a000, 0x846, 0x846, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00012a780, 0xc00026a000, 0x846, 0x846, 0x203000, 0x0, 0x7e3)
        /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e460, 0xc00026a000, 0x846, 0x846, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x69
crypto/tls.(*atLeastReader).Read(0xc0002b8600, 0xc00026a000, 0x846, 0x846, 0x84, 0x1857aa0, 0xc000688a78)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x60
bytes.(*Buffer).ReadFrom(0xc000491e58, 0x18578c0, 0xc0002b8600, 0x409885, 0x1475fe0, 0x15ccaa0)
        /usr/local/go/src/bytes/buffer.go:207 +0xbd
crypto/tls.(*Conn).readFromUntil(0xc000491c00, 0x1859980, 0xc00000e460, 0x5, 0xc00000e460, 0x11)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xf8
crypto/tls.(*Conn).readRecordOrCCS(0xc000491c00, 0x16cda00, 0xc000491d38, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x125
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0xc000491c00, 0xc0008f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x137
bufio.(*Reader).Read(0xc0006562a0, 0xc00074e038, 0x9, 0x9, 0xc000491880, 0x7ff575fa46d0, 0x0)
        /usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x1857720, 0xc0006562a0, 0xc00074e038, 0x9, 0x9, 0x9, 0x7f90f5, 0xc00004410c, 0xc000688e38)
        /usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
        /usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.readFrameHeader(0xc00074e038, 0x9, 0x9, 0x1857720, 0xc0006562a0, 0x0, 0xc000000000, 0x129a2cd26fe1, 0x2587560)
        /go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:237 +0x88
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00074e000, 0xc000044100, 0xc000044100, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000274a80)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1223 +0x168
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0xd15

goroutine 32519 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37c98, 0x72, 0xffffffffffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00057c198, 0x72, 0x8800, 0x8851, 0xffffffffffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00057c180, 0xc000772000, 0x8851, 0x8851, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00057c180, 0xc000772000, 0x8851, 0x8851, 0x203000, 0x0, 0x8844)
        /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0000be0f0, 0xc000772000, 0x8851, 0x8851, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x69
crypto/tls.(*atLeastReader).Read(0xc00033c220, 0xc000772000, 0x8851, 0x8851, 0x2e, 0x1857aa0, 0xc000687a78)
        /usr/local/go/src/crypto/tls/conn.go:761 +0x60
bytes.(*Buffer).ReadFrom(0xc000491058, 0x18578c0, 0xc00033c220, 0x409885, 0x1475fe0, 0x15ccaa0)
        /usr/local/go/src/bytes/buffer.go:207 +0xbd
crypto/tls.(*Conn).readFromUntil(0xc000490e00, 0x1859980, 0xc0000be0f0, 0x5, 0xc0000be0f0, 0x11)
        /usr/local/go/src/crypto/tls/conn.go:783 +0xf8
crypto/tls.(*Conn).readRecordOrCCS(0xc000490e00, 0x16cda00, 0xc000490f38, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:590 +0x125
crypto/tls.(*Conn).readRecord(...)
        /usr/local/go/src/crypto/tls/conn.go:558
crypto/tls.(*Conn).Read(0xc000490e00, 0xc00063c000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
        /usr/local/go/src/crypto/tls/conn.go:1236 +0x137
bufio.(*Reader).Read(0xc0004b3920, 0xc00074e118, 0x9, 0x9, 0xc000484a80, 0x7ff575fa4008, 0x0)
        /usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x1857720, 0xc0004b3920, 0xc00074e118, 0x9, 0x9, 0x9, 0x7f90f5, 0xc0000bc12c, 0xc000687e38)
        /usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
        /usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.readFrameHeader(0xc00074e118, 0x9, 0x9, 0x1857720, 0xc0004b3920, 0x0, 0xc000000000, 0x129b61744648, 0x2587560)
        /go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:237 +0x88
github.com/sorintlab/stolon/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00074e0e0, 0xc0000bc120, 0xc0000bc120, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000274000)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1223 +0x168
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0xd15

goroutine 32618 [IO wait]:
internal/poll.runtime_pollWait(0x7ff573d37d68, 0x72, 0xffffffffffffffff)
        /usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00012a298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00012a280, 0xc0002b7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00012a280, 0xc0002b7000, 0x1000, 0x1000, 0xc0002b9060, 0xc0008bf938, 0x6cb90a)
        /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000e0d0, 0xc0002b7000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/net.go:177 +0x69
net/http.(*connReader).Read(0xc00050a270, 0xc0002b7000, 0x1000, 0x1000, 0x8, 0xc00004ef00, 0x7ff575fa5460)
        /usr/local/go/src/net/http/server.go:787 +0x107
bufio.(*Reader).fill(0xc0006a3860)
        /usr/local/go/src/bufio/bufio.go:100 +0x10f
bufio.(*Reader).ReadSlice(0xc0006a3860, 0x453f0a, 0xc0008bfa30, 0xc0008bfb00, 0x40b880, 0x1, 0x0)
        /usr/local/go/src/bufio/bufio.go:356 +0x3d
bufio.(*Reader).ReadLine(0xc0006a3860, 0xc0008bfb08, 0xc000484380, 0x7ff575fa5460, 0x0, 0x40c008, 0x30)
        /usr/local/go/src/bufio/bufio.go:385 +0x34
net/textproto.(*Reader).readLineSlice(0xc000149320, 0xc00016a500, 0xc00012a280, 0x0, 0x0, 0x42be11)
        /usr/local/go/src/net/textproto/reader.go:55 +0x6f
net/textproto.(*Reader).ReadLine(...)
        /usr/local/go/src/net/textproto/reader.go:36
net/http.readRequest(0xc0006a3860, 0x0, 0xc00016a500, 0x0, 0x0)
        /usr/local/go/src/net/http/request.go:968 +0x8d
net/http.(*conn).readRequest(0xc0001a3900, 0x188a4c0, 0xc000046840, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:967 +0x163
net/http.(*conn).serve(0xc0001a3900, 0x188a4c0, 0xc000046840)
        /usr/local/go/src/net/http/server.go:1819 +0x6ad
created by net/http.(*Server).Serve
        /usr/local/go/src/net/http/server.go:2884 +0x2f4

goroutine 31937 [select]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).sendKeepAliveLoop(0xc000284700, 0x189f1a0, 0xc0003b0020)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:522 +0x330
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).resetRecv
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:439 +0x178

goroutine 31953 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc0004c2be0, 0xc0003fea70, 0x5, 0x5, 0x15218c0, 0xc0006b6c01, 0xc00040e4e0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:158 +0xe5
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0004c2be0, 0xc0003fea70, 0x5, 0x5, 0x819d10, 0xc00040e480, 0xc0008f29b0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:138 +0x127
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0002e52c0, 0xc0003fea70, 0x5, 0x5, 0xc0008f2a6c, 0xc0008f29d8, 0x82abfc)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:436 +0x55
io.ReadAtLeast(0x18591c0, 0xc0002e52c0, 0xc0003fea70, 0x5, 0x5, 0x5, 0x4e05dc, 0xc0002e51d0, 0xc0006b6c60)
        /usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
        /usr/local/go/src/io/io.go:329
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00016a700, 0xc0003fea70, 0x5, 0x5, 0x15a6b60, 0x7ff573d3c150, 0xc0006b6c60)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/transport.go:420 +0xc8
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc0003fea60, 0x400000, 0x21, 0x21, 0x0, 0x0, 0x1876280, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:508 +0x63
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recvAndDecompress(0xc0003fea60, 0xc00016a700, 0x0, 0x0, 0x400000, 0x0, 0x0, 0x0, 0xc0001712f0, 0x21, ...)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:639 +0x4d
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.recv(0xc0003fea60, 0x7ff573a330c8, 0x25a7c48, 0xc00016a700, 0x0, 0x0, 0x15a6b60, 0xc0004c2dc0, 0x400000, 0x0, ...)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/rpc_util.go:684 +0x9b
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc00071c630, 0x15a6b60, 0xc0004c2dc0, 0x0, 0xc0008f2d48, 0xc0004c2dc0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:861 +0xf0
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00071c630, 0x4054fc, 0x203000)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:712 +0x46
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc0004c9b00, 0xc0008f2e38, 0xc0008f2e28, 0x0, 0x38)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:564 +0x29f
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc0004c9b00, 0x15a6b60, 0xc0004c2dc0, 0xc0008f2ef4, 0x2)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:711 +0x100
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb.(*watchWatchClient).Recv(0xc0002e7e10, 0xc0008f2ef0, 0x2, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/etcdserver/etcdserverpb/rpc.pb.go:3217 +0x62
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveWatchClient(0xc00071c580, 0x189f260, 0xc0002e7e10)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:558 +0x5c
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).newWatchClient
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:713 +0x52a

goroutine 31934 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).run(0xc00071c580)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:431 +0x359
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watcher).newWatcherGrpcStream
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:229 +0x2b1

goroutine 31932 [chan receive, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.waitDelete(0x188a4c0, 0xc0001a1240, 0xc0001a4840, 0xc00068c280, 0x43, 0x322be5, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/key.go:33 +0x1e1
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.waitDeletes(0x188a4c0, 0xc0001a1240, 0xc0001a4840, 0xc000128040, 0x33, 0x322be4, 0x1, 0x2, 0xc000166190)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/key.go:62 +0x1da
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency.(*Election).Campaign(0xc00078ff38, 0x188a4c0, 0xc0001a1240, 0xc00030bf20, 0x8, 0xc000128040, 0x33)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/concurrency/election.go:85 +0x6d5
github.com/sorintlab/stolon/internal/store.(*etcdv3Election).campaign(0xc0001220e0)
        /go/src/github.com/sorintlab/stolon/internal/store/etcdv3.go:208 +0x2cf
created by github.com/sorintlab/stolon/internal/store.(*etcdv3Election).RunForElection
        /go/src/github.com/sorintlab/stolon/internal/store/etcdv3.go:161 +0xff

goroutine 32539 [chan receive, 73 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc000202500)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:1067 +0x9ec
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.(*addrConn).connect
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/clientconn.go:684 +0xb6

goroutine 31921 [select]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0001a1000, 0x1, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x104
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000656300, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1b6
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000274a80)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x7b
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0xeb2

goroutine 32520 [select]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0004ce200, 0x1, 0x0, 0x0, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x104
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0004b3ce0, 0x0, 0x0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1b6
github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000274000)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x7b
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0xeb2

goroutine 31954 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).serveSubstream(0xc00071c580, 0xc00071c6e0, 0xc0006b6ae0)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:604 +0x2a0
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*watchGrpcStream).run
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/watch.go:445 +0xf62

goroutine 31866 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).keepAliveCtxCloser(0xc000284700, 0x171f6abe8c2589f0, 0x188a4c0, 0xc0004ce000, 0xc0005f8480)
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:279 +0x10f
created by github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3.(*lessor).KeepAlive
        /go/src/github.com/sorintlab/stolon/vendor/github.com/coreos/etcd/clientv3/lease.go:246 +0x1d1

goroutine 31936 [select, 79 minutes]:
github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream.func5(0xc000446300, 0xc0004c9b00, 0x188a580, 0xc0002e51d0)
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:315 +0xd7
created by github.com/sorintlab/stolon/vendor/google.golang.org/grpc.newClientStream
        /go/src/github.com/sorintlab/stolon/vendor/google.golang.org/grpc/stream.go:314 +0x943

rax    0xca
rbx    0x25889a0
rcx    0x45ae03
rdx    0x0
rdi    0x2588ae8
rsi    0x80
rbp    0x7fff0ff839a8
rsp    0x7fff0ff83960
r8     0x0
r9     0x0
r10    0x0
r11    0x286
r12    0x1
r13    0xc0001d0a80
r14    0xc0001d0a20
r15    0x0
rip    0x45ae01
rflags 0x286
cs     0x33
fs     0x0
gs     0x0