api service panic

trace.log

2015/12/17 10:15:10 API service starting...
2015/12/17 10:15:10 API service running on localhost:15441...
2015/12/17 10:21:15 Failed to read pod manifest for pod "1ffeb0be-88f9-4559-b4d9-06b918313e1c": no such file or directory
2015/12/17 10:21:16 Failed to read pod manifest for pod "1ffeb0be-88f9-4559-b4d9-06b918313e1c": no such file or directory
2015/12/17 10:21:17 Failed to read pod manifest for pod "1ffeb0be-88f9-4559-b4d9-06b918313e1c": no such file or directory
2015/12/17 10:21:17 Failed to read pod manifest for pod "1ffeb0be-88f9-4559-b4d9-06b918313e1c": no such file or directory


























panic: cas db lock already gained

goroutine 552 [running]:
github.com/coreos/rkt/store.(*DB).Open(0xc20814c620, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/store/db.go:48 +0x97
github.com/coreos/rkt/store.(*DB).Do(0xc20814c620, 0xc2082935e0, 0x1, 0x1, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/store/db.go:90 +0x3d
github.com/coreos/rkt/store.(*Store).GetACIInfoWithBlobKey(0xc208062b40, 0xc20825cc30, 0x47, 0x1, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/store/store.go:837 +0x1bb
main.getImageInfo(0xc208062b40, 0xc20825cc30, 0x47, 0x0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:351 +0x65
main.fillAppInfo(0xc208062b40, 0xc2081a8d00, 0xc208232d80, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:375 +0x208
main.func·003(0xc2081a8d00)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:332 +0x100
main.walkPods(0x7f199886581e, 0xc208293af8, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/pods.go:141 +0x661
main.(*v1AlphaAPIServer).ListPods(0xc208034c80, 0x7f1998865bf0, 0xc2081b9e30, 0xc2081adb40, 0xc208034c80, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:339 +0x96
github.com/coreos/rkt/api/v1alpha._PublicAPI_ListPods_Handler(0xc79ee0, 0xc208034c80, 0x7f1998865bf0, 0xc2081b9e30, 0x7f1998864f40, 0x13f6030, 0xc2081a8c40, 0x32, 0x32, 0x0, ...)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/api/v1alpha/api.pb.go:871 +0x173
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.(*Server).processUnaryRPC(0xc208109680, 0x7f1998865a08, 0xc208068e10, 0xc208255740, 0xc2080a9620, 0x13aa758, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:293 +0xa12
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.(*Server).handleStream(0xc208109680, 0x7f1998865a08, 0xc208068e10, 0xc208255740)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:394 +0x6f7
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.func·020(0xc208255740)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:228 +0x41
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.func·008()
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:208 +0x3a
created by github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.(*http2Server).operateHeaders
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:210 +0xc9f

goroutine 1 [chan receive, 5 minutes, locked to thread]:
main.runAPIService(0x13ab5a0, 0x13f6030, 0x0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:625 +0x6c3
main.func·018(0x13ab5a0, 0x13f6030, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/rkt.go:125 +0x4d
github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra.(*Command).execute(0x13ab5a0, 0x13f6030, 0x0, 0x0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra/command.go:572 +0x82f
github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra.(*Command).ExecuteC(0x13ab3a0, 0x13ab5a0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra/command.go:662 +0x4db
github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra.(*Command).Execute(0x13ab3a0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/spf13/cobra/command.go:618 +0x3a
main.main()
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/rkt.go:138 +0x9f

goroutine 13 [IO wait, 1 minutes]:
net.(*pollDesc).Wait(0xc208010d10, 0x72, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208010d10, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).accept(0xc208010cb0, 0x0, 0x7f1998861798, 0xc20813a070)
	/usr/local/go/src/net/fd_unix.go:419 +0x40b
net.(*TCPListener).AcceptTCP(0xc2080346e8, 0xc208018e48, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:234 +0x4e
net.(*TCPListener).Accept(0xc2080346e8, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/tcpsock_posix.go:244 +0x4c
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.(*Server).Serve(0xc208109680, 0x7f1998863ed0, 0xc2080346e8, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:198 +0x1f3
created by main.runAPIService
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:621 +0x5bb

goroutine 8 [syscall, 5 minutes]:
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:21 +0x1f
created by os/signal.init·1
	/usr/local/go/src/os/signal/signal_unix.go:27 +0x35

goroutine 17 [syscall, 5 minutes, locked to thread]:
runtime.goexit()
	/usr/local/go/src/runtime/asm_amd64.s:2232 +0x1

goroutine 14 [select, 1 minutes]:
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.(*http2Server).controller(0xc208068e10)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:611 +0x61f
created by github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.newHTTP2Server
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:133 +0x8d5

goroutine 15 [IO wait]:
net.(*pollDesc).Wait(0xc208074290, 0x72, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:84 +0x47
net.(*pollDesc).WaitRead(0xc208074290, 0x0, 0x0)
	/usr/local/go/src/net/fd_poll_runtime.go:89 +0x43
net.(*netFD).Read(0xc208074230, 0xc208068384, 0x9, 0x9, 0x0, 0x7f1998861798, 0xc208192150)
	/usr/local/go/src/net/fd_unix.go:242 +0x40f
net.(*conn).Read(0xc208034080, 0xc208068384, 0x9, 0x9, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:121 +0xdc
io.ReadAtLeast(0x7f19988659b8, 0xc208034080, 0xc208068384, 0x9, 0x9, 0x9, 0x0, 0x0, 0x0)
	/usr/local/go/src/io/io.go:298 +0xf1
io.ReadFull(0x7f19988659b8, 0xc208034080, 0xc208068384, 0x9, 0x9, 0x37, 0x0, 0x0)
	/usr/local/go/src/io/io.go:316 +0x6d
github.com/coreos/rkt/Godeps/_workspace/src/github.com/bradfitz/http2.readFrameHeader(0xc208068384, 0x9, 0x9, 0x7f19988659b8, 0xc208034080, 0x0, 0xc200000000, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/bradfitz/http2/frame.go:228 +0xa2
github.com/coreos/rkt/Godeps/_workspace/src/github.com/bradfitz/http2.(*Framer).ReadFrame(0xc208068360, 0x0, 0x0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/bradfitz/http2/frame.go:373 +0xf2
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.(*framer).readFrame(0xc20813e3f0, 0x0, 0x0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http_util.go:445 +0x50
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc208068e10, 0xc20814c0a0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:249 +0x689
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.func·021()
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:229 +0xc5
created by github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.(*Server).Serve
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:233 +0x7f8

goroutine 549 [syscall]:
syscall.Syscall(0x4a, 0xf, 0x0, 0x0, 0x7f1998855eb8, 0x30000c20820d268, 0x5)
	/usr/local/go/src/syscall/asm_linux_amd64.s:21 +0x5
syscall.Fsync(0xf, 0x0, 0x0)
	/usr/local/go/src/syscall/zsyscall_linux_amd64.go:503 +0x55
os.(*File).Sync(0xc208035d60, 0x0, 0x0)
	/usr/local/go/src/os/file_posix.go:129 +0x6a
github.com/coreos/rkt/Godeps/_workspace/src/github.com/cznic/ql.(*file).Close(0xc20826b6c0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/cznic/ql/file.go:590 +0xaf
github.com/coreos/rkt/Godeps/_workspace/src/github.com/cznic/ql.(*DB).Close(0xc20826b730, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/cznic/ql/ql.go:1361 +0x13a
github.com/coreos/rkt/Godeps/_workspace/src/github.com/cznic/ql.(*driverConn).Close(0xc2081b97a0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/github.com/cznic/ql/driver.go:237 +0x372
database/sql.(*driverConn).finalClose(0xc208232780, 0x0, 0x0)
	/usr/local/go/src/database/sql/sql.go:322 +0x11e
database/sql.finalCloser.(database/sql.finalClose)·fm(0x0, 0x0)
	/usr/local/go/src/database/sql/sql.go:410 +0x4b
database/sql.(*DB).Close(0xc2080668c0, 0x0, 0x0)
	/usr/local/go/src/database/sql/sql.go:493 +0x3bb
github.com/coreos/rkt/store.(*DB).Close(0xc20814c620, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/store/db.go:72 +0xfd
github.com/coreos/rkt/store.(*DB).Do(0xc20814c620, 0xc20820d5e0, 0x1, 0x1, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/store/db.go:96 +0xdc
github.com/coreos/rkt/store.(*Store).GetACIInfoWithBlobKey(0xc208062b40, 0xc20820fd10, 0x47, 0x1, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/store/store.go:837 +0x1bb
main.getImageInfo(0xc208062b40, 0xc20820fd10, 0x47, 0x0, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:351 +0x65
main.fillAppInfo(0xc208062b40, 0xc2081bb3c0, 0xc2081a3b00, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:375 +0x208
main.func·003(0xc2081bb3c0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:332 +0x100
main.walkPods(0x7f199886581e, 0xc20820daf8, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/pods.go:141 +0x661
main.(*v1AlphaAPIServer).ListPods(0xc208034c80, 0x7f1998865bf0, 0xc2082305d0, 0xc208265cc0, 0xc208034c80, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/rkt/api_service.go:339 +0x96
github.com/coreos/rkt/api/v1alpha._PublicAPI_ListPods_Handler(0xc79ee0, 0xc208034c80, 0x7f1998865bf0, 0xc2082305d0, 0x7f1998864f40, 0x13f6030, 0xc2081bb2c0, 0x34, 0x34, 0x0, ...)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/api/v1alpha/api.pb.go:871 +0x173
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.(*Server).processUnaryRPC(0xc208109680, 0x7f1998865a08, 0xc208068e10, 0xc208254d80, 0xc2080a9620, 0x13aa758, 0x0, 0x0)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:293 +0xa12
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.(*Server).handleStream(0xc208109680, 0x7f1998865a08, 0xc208068e10, 0xc208254d80)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:394 +0x6f7
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc.func·020(0xc208254d80)
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/server.go:228 +0x41
github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.func·008()
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:208 +0x3a
created by github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport.(*http2Server).operateHeaders
	/home/yifan/gopher/src/github.com/coreos/rkt/build-rkt-0.13.0+git/gopath/src/github.com/coreos/rkt/Godeps/_workspace/src/google.golang.org/grpc/transport/http2_server.go:210 +0xc9f