influx crash

influxd.log

runtime/cgo: pthread_create failed: Resource temporarily unavailable
SIGABRT: abort
PC=0x7fc867320165

goroutine 0 [idle]:

goroutine 1 [chan receive, 1440 minutes]:
main.(*Main).Run(0xc20803b700, 0xc20800a010, 0x4, 0x4, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/main.go:96 +0x7a1
main.main()
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/main.go:46 +0xdc

goroutine 6 [syscall, 1440 minutes]:
os/signal.loop()
        /root/.gvm/gos/go1.4.3/src/os/signal/signal_unix.go:21 +0x1f
created by os/signal.init·1
        /root/.gvm/gos/go1.4.3/src/os/signal/signal_unix.go:27 +0x35

goroutine 8 [IO wait, 1440 minutes]:
net.(*pollDesc).Wait(0xc208010610, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208010610, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc2080105b0, 0x0, 0x7fc867a7aee8, 0xc2080d8000)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20802c030, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc20802c030, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:244 +0x4c
github.com/influxdb/influxdb/tcp.(*Mux).Serve(0xc208072660, 0x7fc867a7da18, 0xc20802c030, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tcp/mux.go:52 +0xc7
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:396 +0x25e

goroutine 17 [syscall, 1440 minutes, locked to thread]:
runtime.goexit()
        /root/.gvm/gos/go1.4.3/src/runtime/asm_amd64.s:2232 +0x1

goroutine 9 [chan receive, 1440 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc2080e7c00, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tcp/mux.go:135 +0x80
github.com/influxdb/influxdb/services/meta.(*raftLayer).Accept(0xc208063320, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/raft_state.go:325 +0x65
github.com/hashicorp/raft.(*NetworkTransport).listen(0xc208010690)
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/net_transport.go:362 +0x50
created by github.com/hashicorp/raft.NewNetworkTransportWithLogger
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/net_transport.go:154 +0x2c4

goroutine 10 [select]:
github.com/hashicorp/raft.(*Raft).leaderLoop(0xc2080bc380)
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:874 +0xe30
github.com/hashicorp/raft.(*Raft).runLeader(0xc2080bc380)
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:838 +0x876
github.com/hashicorp/raft.(*Raft).run(0xc2080bc380)
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:602 +0xc9
github.com/hashicorp/raft.*Raft.(github.com/hashicorp/raft.run)·fm()
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:252 +0x27
github.com/hashicorp/raft.func·011()
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/state.go:152 +0x51
created by github.com/hashicorp/raft.(*raftState).goFunc
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/state.go:153 +0xe3

goroutine 11 [select, 420 minutes]:
github.com/hashicorp/raft.(*Raft).runFSM(0xc2080bc380)
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:509 +0x1043
github.com/hashicorp/raft.*Raft.(github.com/hashicorp/raft.runFSM)·fm()
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:253 +0x27
github.com/hashicorp/raft.func·011()
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/state.go:152 +0x51
created by github.com/hashicorp/raft.(*raftState).goFunc
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/state.go:153 +0xe3

goroutine 12 [select, 1 minutes]:
github.com/hashicorp/raft.(*Raft).runSnapshots(0xc2080bc380)
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:1705 +0x466
github.com/hashicorp/raft.*Raft.(github.com/hashicorp/raft.runSnapshots)·fm()
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/raft.go:254 +0x27
github.com/hashicorp/raft.func·011()
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/state.go:152 +0x51
created by github.com/hashicorp/raft.(*raftState).goFunc
        /tmp/tmp.r1Nea5VXEw/src/github.com/hashicorp/raft/state.go:153 +0xe3

goroutine 13 [select, 1440 minutes]:
github.com/influxdb/influxdb/services/meta.(*raftState).logLeaderChanges(0xc20807a3f0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/raft_state.go:145 +0x459
created by github.com/influxdb/influxdb/services/meta.(*raftState).open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/raft_state.go:135 +0x12b2

goroutine 15 [IO wait, 1271 minutes]:
net.(*pollDesc).Wait(0xc208010680, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208010680, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc208010620, 0x0, 0x7fc867a7aee8, 0xc20d774e68)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20802c038, 0xc2080d3db8, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc20802c038, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:244 +0x4c
net/http.(*Server).Serve(0xc2081187e0, 0x7fc867a7da18, 0xc20802c038, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1728 +0x92
net/http.Serve(0x7fc867a7da18, 0xc20802c038, 0x7fc867a8ad58, 0xc2080101c0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1606 +0xa1
github.com/influxdb/influxdb/services/meta.(*Service).serve(0xc2080b6800)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/service.go:131 +0x7a
created by github.com/influxdb/influxdb/services/meta.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/service.go:123 +0xa06

goroutine 16 [select, 1440 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Server).monitorErrorChan(0xc2080dc300, 0xc2080723c0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:609 +0x14a
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:404 +0x44c

goroutine 22 [select, 420 minutes]:
net/http.(*persistConn).roundTrip(0xc218682b00, 0xc218e7a760, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:1082 +0x7ad
net/http.(*Transport).RoundTrip(0xc20807a000, 0xc229e3a0d0, 0xc2232b0b40, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:235 +0x558
net/http.send(0xc229e3a0d0, 0x7fc867a7b0b8, 0xc20807a000, 0x20, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/client.go:219 +0x4fc
net/http.(*Client).send(0x1035aa0, 0xc229e3a0d0, 0x20, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/client.go:142 +0x15b
net/http.(*Client).doFollowingRedirects(0x1035aa0, 0xc229e3a0d0, 0xd60258, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/client.go:367 +0xb25
net/http.(*Client).Get(0x1035aa0, 0xc2232b0b20, 0x20, 0xc229e3fdb8, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/client.go:299 +0xba
net/http.Get(0xc2232b0b20, 0x20, 0xc218e7a750, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/client.go:276 +0x50
github.com/influxdb/influxdb/services/meta.(*Client).getSnapshot(0xc2080b6000, 0xc2080e7b30, 0xe, 0x5b8, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/client.go:1075 +0x1d1
github.com/influxdb/influxdb/services/meta.(*Client).retryUntilSnapshot(0xc2080b6000, 0x5b8, 0xc2318b2c60)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/client.go:1130 +0x170
github.com/influxdb/influxdb/services/meta.(*Client).pollForUpdates(0xc2080b6000)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/client.go:1055 +0x40
created by github.com/influxdb/influxdb/services/meta.(*Client).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/client.go:94 +0x101

goroutine 1465890 [IO wait, 420 minutes]:
net.(*pollDesc).Wait(0xc23c8d7100, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc23c8d7100, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc23c8d70a0, 0xc24eacc000, 0x8000, 0x8000, 0x0, 0x7fc867a7aee8, 0xc25d961e20)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc23efb54e8, 0xc24eacc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
io.Copy(0x7fc867a8b218, 0xc212366870, 0x7fc867a8ade0, 0xc23efb54e8, 0x56f, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/io/io.go:362 +0x1f6
net/http.func·008()
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:171 +0x7d
created by net/http.(*conn).closeNotify
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:177 +0x2e2

goroutine 23 [select, 420 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Server).updateMetaNodeInformation(0xc2080dc300)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:680 +0x66d
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Server).initializeMetaClient
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/server.go:630 +0x114

goroutine 2337943 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc24f0a8140)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 26 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc2086fa6e0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 27 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc2086fa6e0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 28 [runnable]:
syscall.Syscall(0x4, 0xc22a0b7770, 0xc3ce3e4000, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/syscall/asm_linux_amd64.s:21 +0x5
syscall.Stat(0xc22a0b7720, 0x48, 0xc3ce3e4000, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/syscall/zsyscall_linux_amd64.go:1723 +0x92
os.Stat(0xc22a0b7720, 0x48, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/os/file_unix.go:134 +0x75
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Tombstoner).HasTombstones(0xc2088b0020, 0xece569406)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tombstone.go:56 +0x4a
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*TSMReader).Stats(0xc20873df20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/reader.go:334 +0x18e
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*FileStore).Stats(0xc2085f99d0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/file_store.go:332 +0x18d
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*DefaultPlanner).findGenerations(0xc20808f3c0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:382 +0xae
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*DefaultPlanner).PlanLevel(0xc20808f3c0, 0x1, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:138 +0x5a
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2086fa6e0, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:492 +0x144
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 29 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2086fa6e0, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 30 [runnable]:
syscall.Syscall(0x4, 0xc2497a5e00, 0xc3ce0be090, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/syscall/asm_linux_amd64.s:21 +0x5
syscall.Stat(0xc2497a5db0, 0x48, 0xc3ce0be090, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/syscall/zsyscall_linux_amd64.go:1723 +0x92
os.Stat(0xc2497a5db0, 0x48, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/os/file_unix.go:134 +0x75
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Tombstoner).HasTombstones(0xc2088b0020, 0xece569406)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/tombstone.go:56 +0x4a
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*TSMReader).Stats(0xc20873df20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/reader.go:334 +0x18e
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*FileStore).Stats(0xc2085f99d0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/file_store.go:332 +0x18d
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*DefaultPlanner).findGenerations(0xc20808f3c0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:382 +0xae
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*DefaultPlanner).PlanLevel(0xc20808f3c0, 0x3, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:138 +0x5a
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2086fa6e0, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:492 +0x144
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec


goroutine 67 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc20933edc0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 68 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc20933edc0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 69 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc20933edc0, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 70 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc20933edc0, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

.

.

.

. repeating lines

.

.

.

goroutine 628 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc21852d0e0, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec

goroutine 629 [select]:
github.com/influxdb/influxdb/tsdb.(*Store).periodicMaintenance(0xc2080942c0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/store.go:366 +0x11b
created by github.com/influxdb/influxdb/tsdb.(*Store).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/store.go:421 +0x387

goroutine 630 [select]:
github.com/influxdb/influxdb/services/hh.(*Service).purgeInactiveProcessors(0xc208036270)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/hh/service.go:227 +0x199
created by github.com/influxdb/influxdb/services/hh.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/hh/service.go:121 +0x601

goroutine 631 [chan receive]:
github.com/influxdb/influxdb/services/subscriber.(*Service).writePoints(0xc2080104d0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/subscriber/service.go:225 +0x99
created by github.com/influxdb/influxdb/services/subscriber.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/subscriber/service.go:85 +0x167

goroutine 632 [select, 420 minutes]:
github.com/influxdb/influxdb/services/subscriber.(*Service).waitForMetaUpdates(0xc2080104d0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/subscriber/service.go:119 +0x2ad
created by github.com/influxdb/influxdb/services/subscriber.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/subscriber/service.go:87 +0x181

goroutine 633 [select]:
github.com/influxdb/influxdb/monitor.(*Monitor).storeStatistics(0xc208084000)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/monitor/service.go:360 +0xf6e
created by github.com/influxdb/influxdb/monitor.(*Monitor).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/monitor/service.go:141 +0x33c

goroutine 634 [chan receive, 1440 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc208129e80, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tcp/mux.go:135 +0x80
github.com/influxdb/influxdb/cluster.(*Service).serve(0xc2080b6680)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/service.go:98 +0x102
created by github.com/influxdb/influxdb/cluster.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/service.go:75 +0x138

goroutine 635 [select]:
github.com/influxdb/influxdb/services/precreator.(*Service).runPrecreation(0xc2081a3d60)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/precreator/service.go:75 +0x3b7
created by github.com/influxdb/influxdb/services/precreator.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/precreator/service.go:53 +0x20a

goroutine 636 [chan receive, 1440 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc208129e90, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tcp/mux.go:135 +0x80
github.com/influxdb/influxdb/services/snapshotter.(*Service).serve(0xc2080c6360)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/snapshotter/service.go:88 +0xad
created by github.com/influxdb/influxdb/services/snapshotter.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/snapshotter/service.go:61 +0x133

goroutine 637 [chan receive, 1440 minutes]:
github.com/influxdb/influxdb/tcp.(*listener).Accept(0xc208129ea0, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tcp/mux.go:135 +0x80
github.com/influxdb/influxdb/services/copier.(*Service).serve(0xc2081a3ea0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/copier/service.go:78 +0xad
created by github.com/influxdb/influxdb/services/copier.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/copier/service.go:51 +0x133

goroutine 638 [IO wait, 1440 minutes]:
net.(*pollDesc).Wait(0xc21d80ced0, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc21d80ced0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc21d80ce70, 0x0, 0x7fc867a7aee8, 0xc2144c0550)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20c64c5d0, 0xc2080f6380, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc20c64c5d0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:244 +0x4c
net/http.(*Server).Serve(0xc218b016e0, 0x7fc867a7da18, 0xc20c64c5d0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1728 +0x92
net/http.Serve(0x7fc867a7da18, 0xc20c64c5d0, 0x7fc7fc28c2a8, 0xc2144c0540, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1606 +0xa1
github.com/influxdb/influxdb/services/admin.(*Service).serve(0xc2081a3f40)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/admin/service.go:107 +0x147
created by github.com/influxdb/influxdb/services/admin.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/admin/service.go:70 +0x472

goroutine 639 [chan receive, 15 minutes]:
github.com/influxdb/influxdb/services/continuous_querier.(*Service).runContinuousQueryAndWriteResult(0xc2080b6780, 0xc211c930e0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:359 +0x241
github.com/influxdb/influxdb/services/continuous_querier.(*Service).ExecuteContinuousQuery(0xc2080b6780, 0xc2cadcf9f0, 0xc2272e3320, 0xece737bbe, 0xc20ff02b1e, 0x1035ba0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:335 +0x9cd
github.com/influxdb/influxdb/services/continuous_querier.(*Service).runContinuousQueries(0xc2080b6780, 0xc2258d28d0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:245 +0x3be
github.com/influxdb/influxdb/services/continuous_querier.(*Service).backgroundLoop(0xc2080b6780)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:207 +0x57f
created by github.com/influxdb/influxdb/services/continuous_querier.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/continuous_querier/service.go:121 +0x291

goroutine 640 [IO wait]:
net.(*pollDesc).Wait(0xc21d80cf40, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc21d80cf40, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc21d80cee0, 0x0, 0x7fc867a7aee8, 0xc23ffabd90)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc20c64c5d8, 0xc218778dc8, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc20c64c5d8, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/tcpsock_posix.go:244 +0x4c
net/http.(*Server).Serve(0xc212d28c60, 0x7fc867a7da18, 0xc20c64c5d8, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1728 +0x92
net/http.Serve(0x7fc867a7da18, 0xc20c64c5d8, 0x7fc7fc28c230, 0xc2080b6a00, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1606 +0xa1
github.com/influxdb/influxdb/services/httpd.(*Service).serve(0xc2080c6540)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/httpd/service.go:148 +0x79
created by github.com/influxdb/influxdb/services/httpd.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/httpd/service.go:116 +0x73b

goroutine 641 [select]:
github.com/influxdb/influxdb/tsdb.func·002()
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/batcher.go:74 +0x511
created by github.com/influxdb/influxdb/tsdb.(*PointBatcher).Start
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/batcher.go:109 +0x29a

goroutine 642 [IO wait]:
net.(*pollDesc).Wait(0xc21d80d020, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc21d80d020, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).readFrom(0xc21d80cfc0, 0xc21a806000, 0x5ac, 0x5ac, 0x0, 0x0, 0x0, 0x7fc867a7aee8, 0xc3c7501288)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:269 +0x4a1
net.(*UDPConn).ReadFromUDP(0xc20c64cac8, 0xc21a806000, 0x5ac, 0x5ac, 0x522, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/udpsock_posix.go:67 +0x124
github.com/influxdb/influxdb/services/collectd.(*Service).serve(0xc2081fa090)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/collectd/service.go:210 +0x119
created by github.com/influxdb/influxdb/services/collectd.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/collectd/service.go:141 +0x103d

goroutine 643 [select]:
github.com/influxdb/influxdb/services/collectd.(*Service).writePoints(0xc2081fa090)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/collectd/service.go:243 +0x4da
created by github.com/influxdb/influxdb/services/collectd.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/collectd/service.go:142 +0x105a

goroutine 644 [select]:
github.com/influxdb/influxdb/services/retention.(*Service).deleteShardGroups(0xc2080c6660)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/retention/service.go:68 +0xa08
created by github.com/influxdb/influxdb/services/retention.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/retention/service.go:44 +0x19b

goroutine 645 [select]:
github.com/influxdb/influxdb/services/retention.(*Service).deleteShards(0xc2080c6660)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/retention/service.go:102 +0xd03
created by github.com/influxdb/influxdb/services/retention.(*Service).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/retention/service.go:45 +0x1b8

goroutine 646 [select, 1440 minutes]:
github.com/influxdb/influxdb/cmd/influxd/run.(*Command).monitorServerErrors(0xc20807a360)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go:149 +0x2dd
created by github.com/influxdb/influxdb/cmd/influxd/run.(*Command).Run
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cmd/influxd/run/command.go:131 +0xf78

goroutine 2337945 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc24f0a8140, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 693 [IO wait, 401 minutes]:
net.(*pollDesc).Wait(0xc20fd34370, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20fd34370, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc20fd34310, 0xc21a37e000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc23053f1f8)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc209b902f8, 0xc21a37e000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.noteEOFReader.Read(0x7fc867a8ade0, 0xc209b902f8, 0xc218682268, 0xc21a37e000, 0x1000, 0x1000, 0xa6fd20, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:1270 +0x6e
net/http.(*noteEOFReader).Read(0xc219ac9200, 0xc21a37e000, 0x1000, 0x1000, 0xc208012000, 0x0, 0x0)
        <autogenerated>:125 +0xd4
bufio.(*Reader).fill(0xc2131d2600)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).Peek(0xc2131d2600, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:132 +0xf0
net/http.(*persistConn).readLoop(0xc218682210)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:842 +0xa4
created by net/http.(*Transport).dialConn
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:660 +0xc9f

goroutine 2333844 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc208df8820, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 694 [select, 401 minutes]:
net/http.(*persistConn).writeLoop(0xc218682210)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:945 +0x41d
created by net/http.(*Transport).dialConn
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:661 +0xcbc

goroutine 12377353 [IO wait]:
net.(*pollDesc).Wait(0xc244de78e0, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc244de78e0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc244de7880, 0xc2bfe72000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc21b21f268)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc29a81a5c0, 0xc2bfe72000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.(*liveSwitchReader).Read(0xc253ea1c68, 0xc2bfe72000, 0x1000, 0x1000, 0xece73d9f3, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:214 +0xab
io.(*LimitedReader).Read(0xc2f9b95da0, 0xc2bfe72000, 0x1000, 0x1000, 0xc2081de000, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/io/io.go:408 +0xce
bufio.(*Reader).fill(0xc22268c7e0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).ReadSlice(0xc22268c7e0, 0x104040a, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:295 +0x257
bufio.(*Reader).ReadLine(0xc22268c7e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:324 +0x62
net/textproto.(*Reader).readLineSlice(0xc20ce97bc0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:55 +0x9e
net/textproto.(*Reader).ReadLine(0xc20ce97bc0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:36 +0x4f
net/http.ReadRequest(0xc22268c7e0, 0xc218b10820, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/request.go:598 +0xcb
net/http.(*conn).readRequest(0xc253ea1c20, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:586 +0x26f
net/http.(*conn).serve(0xc253ea1c20)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1162 +0x69e
created by net/http.(*Server).Serve
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1751 +0x35e

goroutine 695 [IO wait, 401 minutes]:
net.(*pollDesc).Wait(0xc20fd34450, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc20fd34450, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc20fd343f0, 0xc20b9a0000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc257193230)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc209b90308, 0xc20b9a0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.(*liveSwitchReader).Read(0xc2187964a8, 0xc20b9a0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:214 +0xab
io.(*LimitedReader).Read(0xc219ac9320, 0xc20b9a0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/io/io.go:408 +0xce
bufio.(*Reader).fill(0xc218b388a0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).ReadSlice(0xc218b388a0, 0x10a, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:295 +0x257
bufio.(*Reader).ReadLine(0xc218b388a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:324 +0x62
net/textproto.(*Reader).readLineSlice(0xc220b2f320, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:55 +0x9e
net/textproto.(*Reader).ReadLine(0xc220b2f320, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:36 +0x4f
net/http.ReadRequest(0xc218b388a0, 0xc2dd59f6c0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/request.go:598 +0xcb
net/http.(*conn).readRequest(0xc218796460, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:586 +0x26f
net/http.(*conn).serve(0xc218796460)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1162 +0x69e
created by net/http.(*Server).Serve
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1751 +0x35e

goroutine 6004107 [IO wait, 310 minutes]:
net.(*pollDesc).Wait(0xc25d3ad1e0, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc25d3ad1e0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc25d3ad180, 0xc21f167000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc22080e910)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc22cf86b20, 0xc21f167000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.(*liveSwitchReader).Read(0xc22c10d6c8, 0xc21f167000, 0x1000, 0x1000, 0xece73912b, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:214 +0xab
io.(*LimitedReader).Read(0xc23a0debc0, 0xc21f167000, 0x1000, 0x1000, 0xc2081de000, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/io/io.go:408 +0xce
bufio.(*Reader).fill(0xc2a500f620)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).ReadSlice(0xc2a500f620, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:295 +0x257
bufio.(*Reader).ReadLine(0xc2a500f620, 0x0, 0x0, 0x0, 0xc22bfa7a00, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:324 +0x62
net/textproto.(*Reader).readLineSlice(0xc23c91c2a0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:55 +0x9e
net/textproto.(*Reader).ReadLine(0xc23c91c2a0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:36 +0x4f
net/http.ReadRequest(0xc2a500f620, 0xc2409e8a90, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/request.go:598 +0xcb
net/http.(*conn).readRequest(0xc22c10d680, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:586 +0x26f
net/http.(*conn).serve(0xc22c10d680)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1162 +0x69e
created by net/http.(*Server).Serve
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1751 +0x35e

goroutine 2337946 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc24f0a8140, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 8741984 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc220199ae0, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec

goroutine 2298327 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2092112c0, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 12384216 [select]:
github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard(0xc208065540, 0xc2524583c0, 0xc224938af0, 0x8, 0xc224938af8, 0x5, 0x0, 0xc3c9cb3c30, 0x1, 0x1, ...)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:359 +0x9af
github.com/influxdb/influxdb/cluster.func·001(0xc2524583c0, 0xc224938af0, 0x8, 0xc224938af8, 0x5, 0xc3c9cb3c30, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:258 +0xad
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).WritePoints
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:259 +0x4bd

goroutine 8741980 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc220199ae0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 1465818 [select, 420 minutes]:
github.com/influxdb/influxdb/services/meta.(*handler).serveSnapshot(0xc2080101c0, 0x7fc867a8b110, 0xc210f7dfa0, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:240 +0x538
github.com/influxdb/influxdb/services/meta.*handler.(github.com/influxdb/influxdb/services/meta.serveSnapshot)·fm(0x7fc867a8b110, 0xc210f7dfa0, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:88 +0x45
net/http.HandlerFunc.ServeHTTP(0xc226b0c020, 0x7fc867a8b110, 0xc210f7dfa0, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1265 +0x41
github.com/influxdb/influxdb/services/meta.func·002(0x7fc867a8b038, 0xc210f7df40, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:402 +0x288
net/http.HandlerFunc.ServeHTTP(0xc226b0c040, 0x7fc867a8b038, 0xc210f7df40, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1265 +0x41
github.com/influxdb/influxdb/services/meta.func·003(0x7fc867a8b038, 0xc210f7df40, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:411 +0xba
net/http.HandlerFunc.ServeHTTP(0xc210f7dec0, 0x7fc867a8b038, 0xc210f7df40, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1265 +0x41
github.com/influxdb/influxdb/services/meta.func·004(0x7fc867a8b038, 0xc210f7df40, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:421 +0x179
net/http.HandlerFunc.ServeHTTP(0xc226b0c070, 0x7fc867a8b038, 0xc210f7df40, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1265 +0x41
github.com/influxdb/influxdb/services/meta.func·005(0x7fc867a8b038, 0xc210f7df20, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:429 +0x123
net/http.HandlerFunc.ServeHTTP(0xc210f7dee0, 0x7fc867a8b038, 0xc210f7df20, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1265 +0x41
github.com/influxdb/influxdb/services/meta.func·007(0x7fc867a8b000, 0xc2232b2000, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:448 +0x197
net/http.HandlerFunc.ServeHTTP(0xc210f7df00, 0x7fc867a8b000, 0xc2232b2000, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1265 +0x41
github.com/influxdb/influxdb/services/meta.(*handler).ServeHTTP(0xc2080101c0, 0x7fc867a8b000, 0xc2232b2000, 0xc265733ba0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/services/meta/handler.go:88 +0x358
net/http.serverHandler.ServeHTTP(0xc2081187e0, 0x7fc867a8b000, 0xc2232b2000, 0xc265733ba0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1703 +0x19a
net/http.(*conn).serve(0xc216eb6320)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1204 +0xb57
created by net/http.(*Server).Serve
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1751 +0x35e
goroutine 1461290 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc2290dc000)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 12245125 [select]:
github.com/influxdb/influxdb/cluster.(*PointsWriter).WritePoints(0xc208065540, 0xc3cb545180, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:279 +0x774
github.com/influxdb/influxdb/cluster.(*PointsWriter).WritePointsInto(0xc208065540, 0xc3cb545140, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:230 +0xe1
github.com/influxdb/influxdb/tsdb.(*QueryExecutor).writeInto(0xc208072420, 0xc2160130e0, 0xc2c4506240, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/query_executor.go:779 +0x2d5
github.com/influxdb/influxdb/tsdb.(*QueryExecutor).executeStatement(0xc208072420, 0x0, 0x7fc7fc292f30, 0xc2c4506240, 0xc245919418, 0x8, 0xc2183f0d20, 0x0, 0xc2183f0cc0, 0x0, ...)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/query_executor.go:721 +0x221
github.com/influxdb/influxdb/tsdb.func·024()
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/query_executor.go:168 +0xa07
created by github.com/influxdb/influxdb/tsdb.(*QueryExecutor).ExecuteQuery
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/query_executor.go:225 +0x242

goroutine 1461040 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc238409180, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 1465871 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc238408aa0, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 8286662 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2362705a0, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec

goroutine 1461041 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc238409180, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec

goroutine 2298326 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2092112c0, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 1465869 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc238408aa0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 1461291 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc2290dc000)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 2337944 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc24f0a8140)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 2337947 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc24f0a8140, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec

goroutine 12381289 [runnable]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*FloatValue).Value(0xc3ce6c8720, 0xece710a23, 0xc32e4415f0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/encoding.go:237
github.com/influxdb/influxdb/tsdb/engine/tsm1.encodeFloatBlock(0x0, 0x0, 0x0, 0xc3ce6cc000, 0x1b8, 0x300, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/encoding.go:266 +0x23f
github.com/influxdb/influxdb/tsdb/engine/tsm1.Values.Encode(0xc3ce6cc000, 0x1b8, 0x300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/encoding.go:92 +0x2a1
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*tsmKeyIterator).chunk(0xc3a062dba0, 0xc2819a26a8, 0x1, 0x1, 0xc3ce6cc000, 0x1b8, 0x300, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:859 +0x572
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*tsmKeyIterator).combine(0xc3a062dba0, 0xc3ce677800, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:836 +0xb69
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*tsmKeyIterator).Next(0xc3a062dba0, 0xc20b1e9543)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:770 +0xedc
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Compactor).write(0xc216bb1b90, 0xc3a1f3ef50, 0x44, 0x7fc7f8d076c8, 0xc3a062dba0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:558 +0x302
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Compactor).writeNewFiles(0xc216bb1b90, 0x44, 0x4, 0x7fc7f8d076c8, 0xc3a062dba0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:510 +0x364
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Compactor).compact(0xc216bb1b90, 0xc216bb6200, 0xc3a1ce5a80, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:476 +0x5eb
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Compactor).CompactFull(0xc216bb1b90, 0xc3a1ce5a80, 0x3, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/compact.go:481 +0x84
github.com/influxdb/influxdb/tsdb/engine/tsm1.func·006(0x0, 0xc3a1ce5a80, 0x3, 0x4)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:574 +0x453
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:592 +0x2e3

goroutine 8286661 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2362705a0, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 1461037 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc238409180)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 8286658 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc2362705a0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e

goroutine 1461292 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2290dc000, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 1461038 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc238409180)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 1465851 [select, 420 minutes]:
net/http.(*persistConn).writeLoop(0xc218682b00)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:945 +0x41d
created by net/http.(*Transport).dialConn
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:661 +0xcbc

goroutine 8286659 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc2362705a0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 8741983 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc220199ae0, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1


goroutine 1461039 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc238409180, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 1465850 [IO wait, 420 minutes]:
net.(*pollDesc).Wait(0xc24b85c6f0, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc24b85c6f0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc24b85c690, 0xc24cb11000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc22f168820)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc212366660, 0xc24cb11000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.noteEOFReader.Read(0x7fc867a8ade0, 0xc212366660, 0xc218682b58, 0xc24cb11000, 0x1000, 0x1000, 0xa6fd20, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:1270 +0x6e
net/http.(*noteEOFReader).Read(0xc229c43c60, 0xc24cb11000, 0x1000, 0x1000, 0xc208012000, 0x0, 0x0)
        <autogenerated>:125 +0xd4
bufio.(*Reader).fill(0xc2209ad800)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).Peek(0xc2209ad800, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:132 +0xf0
net/http.(*persistConn).readLoop(0xc218682b00)
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:842 +0xa4
created by net/http.(*Transport).dialConn
        /root/.gvm/gos/go1.4.3/src/net/http/transport.go:660 +0xc9f

goroutine 1465872 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc238408aa0, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 12245126 [select]:
github.com/influxdb/influxdb/tsdb.(*AggregateExecutor).execute(0xc22680e780, 0xc2183f0d80, 0xc2183f0cc0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/aggregate.go:173 +0x1c53
created by github.com/influxdb/influxdb/tsdb.(*AggregateExecutor).Execute
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/aggregate.go:48 +0x64

goroutine 1461293 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2290dc000, 0x1, 0x2)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:138 +0x2c1

goroutine 1461294 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2290dc000, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec


goroutine 9675488 [IO wait, 310 minutes]:
net.(*pollDesc).Wait(0xc234f4f720, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc234f4f720, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc234f4f6c0, 0xc237550000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc249557dc0)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc21fa75970, 0xc237550000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.(*liveSwitchReader).Read(0xc2421162c8, 0xc237550000, 0x1000, 0x1000, 0xece73912b, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:214 +0xab
io.(*LimitedReader).Read(0xc240ea8200, 0xc237550000, 0x1000, 0x1000, 0xc2081de000, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/io/io.go:408 +0xce
bufio.(*Reader).fill(0xc25822ac60)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).ReadSlice(0xc25822ac60, 0xa, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:295 +0x257
bufio.(*Reader).ReadLine(0xc25822ac60, 0x0, 0x0, 0x0, 0xc206517000, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:324 +0x62
net/textproto.(*Reader).readLineSlice(0xc222e8f560, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:55 +0x9e
net/textproto.(*Reader).ReadLine(0xc222e8f560, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:36 +0x4f
net/http.ReadRequest(0xc25822ac60, 0xc2409e9040, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/request.go:598 +0xcb
net/http.(*conn).readRequest(0xc242116280, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:586 +0x26f
net/http.(*conn).serve(0xc242116280)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1162 +0x69e
created by net/http.(*Server).Serve
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1751 +0x35e

goroutine 1465873 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc238408aa0, 0x0, 0x3)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:139 +0x2ec

goroutine 1465870 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc238408aa0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 2333843 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMFull(0xc208df8820)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:559 +0x16a
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:136 +0x26b

goroutine 8286660 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactTSMLevel(0xc2362705a0, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:495 +0x17e
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:137 +0x296

goroutine 2333842 [sleep]:
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).compactCache(0xc208df8820)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:466 +0x1f7
created by github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).Open
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:135 +0x24e


goroutine 12377428 [IO wait]:
net.(*pollDesc).Wait(0xc2f5be9480, 0x72, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc2f5be9480, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc2f5be9420, 0xc28a14f000, 0x1000, 0x1000, 0x0, 0x7fc867a7aee8, 0xc22e14d8e0)
        /root/.gvm/gos/go1.4.3/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc269946d50, 0xc28a14f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/net.go:121 +0xdc
net/http.(*liveSwitchReader).Read(0xc226fc4cc8, 0xc28a14f000, 0x1000, 0x1000, 0xece73d9f3, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:214 +0xab
io.(*LimitedReader).Read(0xc2c4f4dbc0, 0xc28a14f000, 0x1000, 0x1000, 0xc2081de000, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/io/io.go:408 +0xce
bufio.(*Reader).fill(0xc2b2622000)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:97 +0x1ce
bufio.(*Reader).ReadSlice(0xc2b2622000, 0x104040a, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:295 +0x257
bufio.(*Reader).ReadLine(0xc2b2622000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/bufio/bufio.go:324 +0x62
net/textproto.(*Reader).readLineSlice(0xc339f5da70, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:55 +0x9e
net/textproto.(*Reader).ReadLine(0xc339f5da70, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/textproto/reader.go:36 +0x4f
net/http.ReadRequest(0xc2b2622000, 0xc25c994820, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/request.go:598 +0xcb
net/http.(*conn).readRequest(0xc226fc4c80, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:586 +0x26f
net/http.(*conn).serve(0xc226fc4c80)
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1162 +0x69e
created by net/http.(*Server).Serve
        /root/.gvm/gos/go1.4.3/src/net/http/server.go:1751 +0x35e

goroutine 12384217 [runnable]:
syscall.Syscall(0x4a, 0x48, 0x0, 0x0, 0x0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/syscall/asm_linux_amd64.s:21 +0x5
syscall.Fsync(0x48, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/syscall/zsyscall_linux_amd64.go:503 +0x55
os.(*File).Sync(0xc20e3b89d0, 0x0, 0x0)
        /root/.gvm/gos/go1.4.3/src/os/file_posix.go:129 +0x6a
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*WALSegmentWriter).sync(0xc2b06f5a00, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:597 +0x6e
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*WAL).writeToLog(0xc22cc11a80, 0x7fc85ae30c98, 0xc22cf87b50, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:243 +0x65e
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*WAL).WritePoints(0xc22cc11a80, 0xc3cd608960, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/wal.go:152 +0xa3
github.com/influxdb/influxdb/tsdb/engine/tsm1.(*Engine).WritePoints(0xc24f0a8140, 0xc3c9cb3c30, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/engine/tsm1/engine.go:325 +0x55f
github.com/influxdb/influxdb/tsdb.(*Shard).WritePoints(0xc24888f760, 0xc3c9cb3c30, 0x1, 0x1, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/shard.go:263 +0x66d
github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard(0xc2080942c0, 0x21f, 0xc3c9cb3c30, 0x1, 0x1, 0x0, 0x0)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/tsdb/store.go:442 +0x14e
github.com/influxdb/influxdb/cluster.func·002(0x21f, 0x1, 0xc3c9cb3c30, 0x1, 0x1)
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:316 +0x103
created by github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard
        /tmp/tmp.r1Nea5VXEw/src/github.com/influxdb/influxdb/cluster/points_writer.go:352 +0x31a


rax     0x0
rbx     0x7fc867673d98
rcx     0xffffffffffffffff
rdx     0x6
rdi     0x4030
rsi     0x4031
rbp     0xd6490e
rsp     0x7fc8672ec928
r8      0xa
r9      0x7fc8672ed700
r10     0x8
r11     0x206
r12     0x223b4f0
r13     0x7fc8672ed9c0
r14     0x7fc867ab5040
r15     0x3
rip     0x7fc867320165
rflags  0x206
cs      0x33
fs      0x0
gs      0x0