kev009

Jun  5 21:42:01 beastie kernel: lock order reversal:
Jun  5 21:42:01 beastie kernel: 1st 0xfffff800b62fb8e0 filedesc structure (filedesc structure) @ /usr/home/mmacy/devel/freebsd-base-graphics/sys/kern/kern_descrip.c:1230
Jun  5 21:42:01 beastie kernel: 2nd 0xfffff8000766a5f0 zfs (zfs) @ /usr/home/mmacy/devel/freebsd-base-graphics/sys/kern/vfs_subr.c:4885
Jun  5 21:42:01 beastie kernel: stack backtrace:
Jun  5 21:42:01 beastie kernel: #0 0xffffffff80aa6ff0 at witness_debugger+0x70
Jun  5 21:42:01 beastie kernel: #1 0xffffffff80aa6ee4 at witness_checkorder+0xe54
Jun  5 21:42:01 beastie kernel: #2 0xffffffff80a1fb56 at __lockmgr_args+0x4d6
Jun  5 21:42:01 beastie kernel: #3 0xffffffff80af67bc at vop_stdlock+0x3c
Jun  5 21:42:01 beastie kernel: #4 0xffffffff81013ca0 at VOP_LOCK1_APV+0x100
Jun  5 21:42:01 beastie kernel: #5 0xffffffff80b1763a at _vn_lock+0x9a

kev009

Loaded symbols for /boot/kernel/linuxkpi.ko
#0  doadump (textdump=1) at pcpu.h:221
221     pcpu.h: No such file or directory.
        in pcpu.h
(kgdb) #0  doadump (textdump=1) at pcpu.h:221
#1  0xffffffff80a409e5 in kern_reboot (howto=<value optimized out>)
    at /usr/home/mmacy/devel/freebsd-base-graphics/sys/kern/kern_shutdown.c:366
#2  0xffffffff80a40fbb in vpanic (fmt=<value optimized out>, 
    ap=<value optimized out>)
    at /usr/home/mmacy/devel/freebsd-base-graphics/sys/kern/kern_shutdown.c:767

kev009

It appears we have a new v6 RA core that is in the code were locking was added, nd6_ra_input.  We are seeing this on em/igb/ix, so it doesn't seem hardware related.

(kgdb) bt
#0  doadump (textdump=1) at pcpu.h:219
#1  0xffffffff807900b7 in kern_reboot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:486
#2  0xffffffff807904b5 in vpanic (fmt=<value optimized out>, ap=<value optimized out>) at /usr/src/sys/kern/kern_shutdown.c:889
#3  0xffffffff80790343 in panic (fmt=0x0) at /usr/src/sys/kern/kern_shutdown.c:818
#4  0xffffffff80b1d91b in trap_fatal (frame=<value optimized out>, eva=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:858
#5  0xffffffff80b1dc1d in trap_pfault (frame=0xfffffe05d59f2ff0, usermode=<value optimized out>) at /usr/src/sys/amd64/amd64/trap.c:681
#6  0xffffffff80b1d29a in trap (frame=0xfffffe05d59f2ff0) at /usr/src/sys/amd64/amd64/trap.c:447

kev009

static int
tcp_sysctl_rfc3390_handler(SYSCTL_HANDLER_ARGS)
{
        int err;
        int new;

        err = sysctl_handle_int(oidp, &new, 0, req);
        if (err || !req->newptr)
                return (err);

kev009

phxnetapp01::> run -node phxnetapp01-1 -command sysconfig -a
*** This system has taken over phxnetapp01-2
        NetApp Release 8.3X13: Tue May 20 01:12:25 PDT 2014
        System ID: 1574272875 (phxnetapp01-1); partner ID: 2016929901 (phxnetapp01-2)
        System Serial Number: 700000845951 (phxnetapp01-1); partner Serial Number: 700000845951 (phxnetapp01-2)
        System Rev: F3
        System Storage Configuration: Mixed-Path HA
        System ACP Connectivity: NA
        Backplane Part Number: 111-00587
        Backplane Rev: D0

kev009

# CC=/usr/local/gcc4/bin/gcc CXX=/usr/local/gcc4/bin/g++ ./bootstrap --abi 32 --compiler gcc
===> bootstrap command: ./bootstrap --abi 32 --compiler gcc
===> bootstrap started: Sun Jun 19 00:54:18 MST 2011
Working directory is: /var/adm/pkgsrc/bootstrap/work
===> running: /usr/bin/sed -e 's|@DEFAULT_INSTALL_MODE@|'0755'|' /var/adm/pkgsrc/sysutils/install-sh/files/install-sh.in > /var/adm/pkgsrc/bootstrap/work/bin/install-sh
===> running: /usr/bin/chmod +x /var/adm/pkgsrc/bootstrap/work/bin/install-sh
===> Creating default mk.conf in /var/adm/pkgsrc/bootstrap/work
===> running: /usr/bin/sh /var/adm/pkgsrc/bootstrap/work/bin/install-sh -d -o root -g sys /var/adm/pkgsrc/bootstrap/work/sbin
===> running: /usr/bin/sh /var/adm/pkgsrc/bootstrap/work/bin/install-sh -d -o root -g sys /var/adm/pkgsrc/bootstrap/work/share/mk
===> Bootstrapping mk-files

kev009

phxnetapp01-1% kldstat -v
Id Refs Address            Size     Name
 1  105 0xffffffff80100000 b83068   kernel (X86_64/freebsd/image1/kernel)
        Contains modules:
                Id Name
                120 hwpmc
                205 if_lo
                84 ipfilter
                198 shell
                197 elf64

kev009

--- contention:
cycles/second=2400135606
78967850741292 32690 @ 0x43e462 0x903de7 0x460831
#	0x43e462	runtime.selectgo+0x12							/usr/local/go/src/runtime/select.go:212
#	0x903de7	github.com/influxdb/influxdb/tsdb/engine/wal.(*Log).autoflusher+0x567	/usr/ports/databases/influxdb/work/influxdb-0.9.4/src/github.com/influxdb/influxdb/tsdb/engine/wal/wal.go:604

13774586346477 34 @ 0x4ff008 0x61385c 0x630588 0x460831
#	0x4ff008	sync.(*RWMutex).RLock+0x58							/usr/local/go/src/sync/rwmutex.go:42
#	0x61385c	github.com/influxdb/influxdb/tsdb.(*Store).WriteToShard+0x3c			/usr/ports/databases/influxdb/work/influxdb-0.9.4/src/github.com/influxdb/influxdb/tsdb/store.go:332
#	0x630588	github.com/influxdb/influxdb/cluster.(*PointsWriter).writeToShard.func1+0x108	/usr/ports/databases/influxdb/work/influxdb-0.9.4/src/github.com/influxdb/influxdb/cluster/points_writer.go:267

kev009

cpboot> tftpboot kernel
Using xlr_gmac3 device
TFTP from server 192.168.12.110; our IP address is 192.168.12.89
Filename 'kernel'.
Load address: 0x87000000
Loading: #################################################################
         ############################
done
Bytes transferred = 6060165 (5c7885 hex)
cpboot> bootelf

kev009

devclass_find_driver_internal:1343: xlr_i2c in devclass iodi
device_probe_child:2071: Trying xlr_i2c
xlr_i2c0: <XLR/XLS I2C bus controller> on iodi0
xlr_i2c0: unused iicbus instance
devclass_find_driver_internal:1343: xlr_i2c in devclass iodi
device_probe_child:2071: Trying xlr_i2c
xlr_i2c1: <XLR/XLS I2C bus controller> on iodi0
iodi0: MEMORY resource - for xlr_i2c1 0-ffffffff
device_add_child_ordered:1866: iicbus at xlr_i2c with order 0 as unit -1
make_device:1751: iicbus at xlr_i2c as unit -1