Skip to content

Instantly share code, notes, and snippets.

@genneko
Last active February 7, 2019 07:31
Show Gist options
  • Save genneko/8b7f8eda3b9048d408ce53e7cb3aaec5 to your computer and use it in GitHub Desktop.
Save genneko/8b7f8eda3b9048d408ce53e7cb3aaec5 to your computer and use it in GitHub Desktop.
Trying to debug kernel for the first time in my life :P, with the help of gdb's remote debugging facility.
define p_v6elem
printf "%02x%02x", $arg0 >> 0 & 0xff, $arg0 >> 8 & 0xff
end
define p_v4
printf "%d.%d.%d.%d", $arg0 >> 0 & 0xff, $arg0 >> 8 & 0xff, $arg0 >> 16 & 0xff, $arg0 >> 24 & 0xff
end
define p_dl
printf "datalink type = %d, nlen = %d, alen = %d", $arg0->sdl_type, $arg0->sdl_nlen, $arg0->sdl_alen
end
define p_sockaddr
if $arg0->sa_family == 2
set $x = ((struct sockaddr_in *)$arg0)->sin_addr.s_addr
p_v4 $x
printf "\n"
else
if $arg0->sa_family == 28
set $x = ((struct sockaddr_in6 *)$arg0)->sin6_addr.__u6_addr
p_v6elem $x.__u6_addr16[0]
printf ":"
p_v6elem $x.__u6_addr16[1]
printf ":"
p_v6elem $x.__u6_addr16[2]
printf ":"
p_v6elem $x.__u6_addr16[3]
printf ":"
p_v6elem $x.__u6_addr16[4]
printf ":"
p_v6elem $x.__u6_addr16[5]
printf ":"
p_v6elem $x.__u6_addr16[6]
printf ":"
p_v6elem $x.__u6_addr16[7]
printf "\n"
else
if $arg0->sa_family == 18
set $x = ((struct sockaddr_dl *)$arg0)
p_dl $x
printf "\n"
else
printf "unknown address family = %d (len = %d)\n", $arg0->sa_family, $arg0->sa_len
end
end
end
end
define p_ifa_list
set $n = $arg0
while $n
p $n->ifa_addr
p *$n->ifa_addr
set $n = $n->ifa_link->cstqe_next
end
end
define p_ifp_addrhead
printf "ifp->if_addrhead = "
p $arg0
printf "ifp->if_addrhead->cstqh_first = "
p $arg0->cstqh_first
set $n = $arg0->cstqh_first
while $n
printf "ifp->if_addrhead->cstqh_first = "
p *$n
set $n = $n->ifa_link->cstqe_next
end
end
define p_ifp_ifa
set $ifp = $arg0
set $ifa = $arg1
if ! $ifp
set $ifp = $ifa->ifa_ifp
end
printf "ifp = "
p $ifp
printf "ifp->if_flags = %08x\n", $ifp->if_flags
if $ifp->if_flags & 0x200000
print "DYING..."
end
printf "ifp->if_addr = "
p $ifp->if_addr
printf "ifa = "
p $ifa
print "[List of ifp->if_addrhead]"
p_ifp_addrhead $ifp->if_addrhead
print "[List of ifp->if_addr]"
p_ifa_list $ifp->if_addr
print "[List of ifa]"
p_ifa_list $ifa
end

First, it always happens in in6_unlink_ifa

(kgdb) f 11
#11 0xffffffff80dff153 in in6_unlink_ifa (ia=0xfffff80013231000,
    ifp=0xfffff80003c4d800) at /usr/src/sys/netinet6/in6.c:1326
1326            CK_STAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifaddr, ifa_link);

(kgdb) l in6_unlink_ifa
1316            in6_unlink_ifa(ia, ifp);
1317    }
1318
1319    static void
1320    in6_unlink_ifa(struct in6_ifaddr *ia, struct ifnet *ifp)
1321    {
1322            char ip6buf[INET6_ADDRSTRLEN];
1323            int remove_lle;
1324
1325            IF_ADDR_WLOCK(ifp);
(kgdb)
1326            CK_STAILQ_REMOVE(&ifp->if_addrhead, &ia->ia_ifa, ifaddr, ifa_link);
1327            IF_ADDR_WUNLOCK(ifp);
1328            ifa_free(&ia->ia_ifa);                  /* if_addrhead */

What does CK_STAILQ_REMOVE macro do?

The macro CK_STAILQ_REMOVE traces the address list starting from head (&ifp->if_addrhead)'s cstqh_first pointer and remove the element if it finds the one matches with the elm (&ia->ia_ifa).

[src/sys/contrib/ck/include/ck_queue.h]

#define CK_STAILQ_REMOVE(head, elm, type, field) do {                           \
        if ((head)->cstqh_first == (elm)) {                                     \
                CK_STAILQ_REMOVE_HEAD((head), field);                           \
        } else {                                                                \
                struct type *curelm = (head)->cstqh_first;                      \
                while (curelm->field.cstqe_next != (elm))                       \
                        curelm = curelm->field.cstqe_next;                      \
                CK_STAILQ_REMOVE_AFTER(head, curelm, field);                    \
        }                                                                       \
} while (0)


#define CK_STAILQ_REMOVE_AFTER(head, elm, field) do {                           \
        ck_pr_store_ptr(&(elm)->field.cstqe_next,                               \
            (elm)->field.cstqe_next->field.cstqe_next);                         \
        if ((elm)->field.cstqe_next == NULL)                                    \
                (head)->cstqh_last = &(elm)->field.cstqe_next;                  \
} while (0)

The macro CK_STAILQ_REMOVE_HEAD deletes the first element of the list pointed by head->cstqh_first (&ifp->if_addrhead) by re-assigning the next element (head->cstqh_first->field(ifa_link).cstqe_next) to head->cstqh_first. If the new first element is NULL, it nullifies the last element (head->cstqh_last) too.

#define CK_STAILQ_REMOVE_HEAD(head, field) do {                                 \
        ck_pr_store_ptr(&(head)->cstqh_first,                                   \
            (head)->cstqh_first->field.cstqe_next);                             \
        if ((head)->cstqh_first == NULL)                                                \
                (head)->cstqh_last = &(head)->cstqh_first;                      \
} while (0)

After patching in6_unlink_ifa to check IFF_DYING, rtsock_addrmsg() causes panics

(kgdb) f 11
#11 rtsock_addrmsg (cmd=2, ifa=0xfffff80019e3dc00, fibnum=-1)
    at /usr/src/sys/net/rtsock.c:1337
1337            info.rti_info[RTAX_IFP] = ifp->if_addr->ifa_addr;

(kgdb) p *ifp
Cannot access memory at address 0x0

(kgdb) l rtsock_addrmsg
1316     * Assume input data to be valid.
1317     * Returns 0 on success.
1318     */
1319    int
1320    rtsock_addrmsg(int cmd, struct ifaddr *ifa, int fibnum)
1321    {
1322            struct rt_addrinfo info;
1323            struct sockaddr *sa;
1324            int ncmd;
1325            struct mbuf *m;
(kgdb)
1326            struct ifa_msghdr *ifam;
1327            struct ifnet *ifp = ifa->ifa_ifp;
1328            struct sockaddr_storage ss;
1329
1330            if (V_route_cb.any_count == 0)
1331                    return (0);
1332
1333            ncmd = cmd == RTM_ADD ? RTM_NEWADDR : RTM_DELADDR;
1334
1335            bzero((caddr_t)&info, sizeof(info));
(kgdb)
1336            info.rti_info[RTAX_IFA] = sa = ifa->ifa_addr;
1337            info.rti_info[RTAX_IFP] = ifp->if_addr->ifa_addr;
1338            info.rti_info[RTAX_NETMASK] = rtsock_fix_netmask(
1339                info.rti_info[RTAX_IFP], ifa->ifa_netmask, &ss);
1340            info.rti_info[RTAX_BRD] = ifa->ifa_dstaddr;
1341            if ((m = rtsock_msg_mbuf(ncmd, &info)) == NULL)
1342                    return (ENOBUFS);
1343            ifam = mtod(m, struct ifa_msghdr *);
1344            ifam->ifam_index = ifp->if_index;
1345            ifam->ifam_metric = ifa->ifa_ifp->if_metric;
(kgdb) p *ifa
$2 = {ifa_addr = 0x0, ifa_dstaddr = 0x0, ifa_netmask = 0x0, ifa_ifp = 0x0,
  ifa_carp = 0x0, ifa_link = {cstqe_next = 0x0}, ifa_rtrequest = 0x0,
  ifa_flags = 0, ifa_refcnt = 4294967295, ifa_ipackets = 0x0,
  ifa_opackets = 0x0, ifa_ibytes = 0x0, ifa_obytes = 0x0, ifa_epoch_ctx = {
    data = {0xffffffff80cb0e60 <ifa_destroy>, 0x0}}}

sometimes, it looks like this.

(kgdb) p *ifa
$87 = {ifa_addr = 0xfffff8003f3a5d00, ifa_dstaddr = 0xfffff80000000001,
  ifa_netmask = 0xfffffe001d895650, ifa_ifp = 0xfffff80000000004,
  ifa_carp = 0xfffffe001d895650, ifa_link = {cstqe_next = 0x4},
  ifa_rtrequest = 0xd8dd39000000000, ifa_flags = 0, ifa_refcnt = 0,
  ifa_ipackets = 0xfffffe001baddaf0, ifa_opackets = 0xfffffe0000443f30,
  ifa_ibytes = 0xfffffe001baddad0, ifa_obytes = 0xfffffe001baddae0,
  ifa_epoch_ctx = {data = {0xffffffff80cb0e60 <ifa_destroy>, 0x0}}}

(kgdb) p *ifa->ifa_addr
$89 = {sa_len = 0 '\000', sa_family = 0 '\000',
  sa_data = '\000' <repeats 13 times>}

(kgdb) p ifa->ifa_ifp->if_xname
$90 = "\324\357\000\360\244\360\000\360\362\346\000\360n\376", <incomplete sequence \360>

#13 in6_purgeaddrs()

(kgdb) up
#13 in6_purgeaddr (ifa=0xfffff80019e3dc00) at /usr/src/sys/netinet6/in6.c:1315
1315            in6_newaddrmsg(ia, RTM_DELETE);
(kgdb) p *ifa
$6 = {ifa_addr = 0x0, ifa_dstaddr = 0x0, ifa_netmask = 0x0, ifa_ifp = 0x0,
  ifa_carp = 0x0, ifa_link = {cstqe_next = 0x0}, ifa_rtrequest = 0x0,
  ifa_flags = 0, ifa_refcnt = 4294967295, ifa_ipackets = 0x0,
  ifa_opackets = 0x0, ifa_ibytes = 0x0, ifa_obytes = 0x0, ifa_epoch_ctx = {
    data = {0xffffffff80cb0e60 <ifa_destroy>, 0x0}}}

(kgdb) list
1310                            log(LOG_INFO, "%s: err=%d, destination address delete "
1311                                "failed\n", __func__, error);
1312                    ia->ia_flags &= ~IFA_ROUTE;
1313            }
1314
1315            in6_newaddrmsg(ia, RTM_DELETE);
1316            in6_unlink_ifa(ia, ifp);
1317    }
1318
1319    static void

#14 if_purgeaddrs()

(kgdb) up
#14 0xffffffff80caeaff in if_purgeaddrs (ifp=0xfffff80003463800)
    at /usr/src/sys/net/if.c:995
995                             in6_purgeaddr(ifa);

(kgdb) p *ifa
$7 = {ifa_addr = 0x0, ifa_dstaddr = 0x0, ifa_netmask = 0x0, ifa_ifp = 0x0,
  ifa_carp = 0x0, ifa_link = {cstqe_next = 0x0}, ifa_rtrequest = 0x0,
  ifa_flags = 0, ifa_refcnt = 4294967295, ifa_ipackets = 0x0,
  ifa_opackets = 0x0, ifa_ibytes = 0x0, ifa_obytes = 0x0, ifa_epoch_ctx = {
    data = {0xffffffff80cb0e60 <ifa_destroy>, 0x0}}}

(kgdb) list
990                                     continue;
991                     }
992     #endif /* INET */
993     #ifdef INET6
994                     if (ifa->ifa_addr->sa_family == AF_INET6) {
995                             in6_purgeaddr(ifa);
996                             /* ifp_addrhead is already updated */
997                             continue;
998                     }
999     #endif /* INET6 */

#15 tunclose()

(kgdb) up
#15 0xffffffff80cc0931 in tunclose (dev=<optimized out>, foo=<optimized out>,
    bar=<optimized out>, td=<optimized out>) at /usr/src/sys/net/if_tun.c:478
478                     if_purgeaddrs(ifp);

(kgdb) p *ifp->if_addr
$16 = {ifa_addr = 0xfffff80019da0b70, ifa_dstaddr = 0x0,
  ifa_netmask = 0xfffff80019da0ba8, ifa_ifp = 0xfffff80003463800,
  ifa_carp = 0x0, ifa_link = {cstqe_next = 0xfffff80019e3dc00},
  ifa_rtrequest = 0xffffffff80cb69a0 <link_rtrequest>, ifa_flags = 0,
  ifa_refcnt = 1, ifa_ipackets = 0xfffffe0000473210,
  ifa_opackets = 0xfffffe0000473200, ifa_ibytes = 0xfffffe0000473230,
  ifa_obytes = 0xfffffe0000473220, ifa_epoch_ctx = {data = {0x0, 0x0}}}

(kgdb) p *ifp
$8 = {if_link = {cstqe_next = 0x0}, if_clones = {le_next = 0x0,
    le_prev = 0xfffff80003474628}, if_groups = {cstqh_first = 0x0,
    cstqh_last = 0xfffff80003463818}, if_alloctype = 23 '\027',
  if_softc = 0xfffff80019905900, if_llsoftc = 0x0, if_l2com = 0x0,
  if_dname = 0xffffffff8168f650 "tun", if_dunit = 0, if_index = 4,
  if_index_reserved = 0, if_xname = "wg0", '\000' <repeats 12 times>,
  if_description = 0x0, if_flags = 2129936, if_drv_flags = 0,
  if_capabilities = 524288, if_capenable = 524288, if_linkmib = 0x0,
  if_linkmiblen = 0, if_refcount = 2, if_type = 23 '\027',
  if_addrlen = 0 '\000', if_hdrlen = 0 '\000', if_link_state = 2 '\002',
  if_mtu = 1420, if_metric = 0, if_baudrate = 0, if_hwassist = 0,
  if_epoch = 76, if_lastchange = {tv_sec = 1549421795, tv_usec = 667363},
  if_snd = {ifq_head = 0x0, ifq_tail = 0x0, ifq_len = 0, ifq_maxlen = 50,
    ifq_mtx = {lock_object = {lo_name = 0xfffff80003463858 "wg0",
        lo_flags = 16973824, lo_data = 0, lo_witness = 0x0}, mtx_lock = 0},
    ifq_drv_head = 0x0, ifq_drv_tail = 0x0, ifq_drv_len = 0,
    ifq_drv_maxlen = 0, altq_type = 0, altq_flags = 1, altq_disc = 0x0,
    altq_ifp = 0xfffff80003463800, altq_enqueue = 0x0, altq_dequeue = 0x0,
    altq_request = 0x0, altq_clfier = 0x0, altq_classify = 0x0,
    altq_tbr = 0x0, altq_cdnr = 0x0}, if_linktask = {ta_link = {
      stqe_next = 0x0}, ta_pending = 0, ta_priority = 0,
    ta_func = 0xffffffff80cadb90 <do_link_state_change>,
    ta_context = 0xfffff80003463800}, if_addr_lock = {lock_object = {
      lo_name = 0xffffffff8123a5b8 "if_addr_lock", lo_flags = 16973824,
      lo_data = 0, lo_witness = 0x0}, mtx_lock = 0}, if_addrhead = {
    cstqh_first = 0xfffff80019da0b00, cstqh_last = 0xfffff80019e3dc28},
  if_multiaddrs = {cstqh_first = 0x0, cstqh_last = 0xfffff800034639b8},
  if_amcount = 0, if_addr = 0xfffff80019da0b00, if_hw_addr = 0x0,
  if_broadcastaddr = 0x0, if_afdata_lock = {lock_object = {
      lo_name = 0xffffffff812b7b6d "if_afdata", lo_flags = 16973824,
      lo_data = 0, lo_witness = 0x0}, mtx_lock = 0}, if_afdata = {0x0, 0x0,
    0xfffff800034c9780, 0x0 <repeats 25 times>, 0xfffff800196b31c0,
    0x0 <repeats 13 times>}, if_afdata_initialized = 2, if_fib = 0,
  if_vnet = 0xfffff80003061d00, if_home_vnet = 0xfffff80003061d00,
  if_vlantrunk = 0x0, if_bpf = 0xffffffff81f9ded8 <dead_bpf_if>,
  if_pcount = 0, if_bridge = 0x0, if_lagg = 0x0, if_pf_kif = 0x0,
  if_carp = 0x0, if_label = 0x0, if_netmap = 0x0,
  if_output = 0xffffffff80cc2020 <tunoutput>,
  if_input = 0xffffffff80cb6930 <if_input_default>, if_bridge_input = 0x0,
  if_bridge_output = 0x0, if_bridge_linkstate = 0x0,
  if_start = 0xffffffff80cc2340 <tunstart>,
  if_ioctl = 0xffffffff80cc1dc0 <tunifioctl>, if_init = 0x0,
  if_resolvemulti = 0x0, if_qflush = 0xffffffff80cb1e40 <if_qflush>,
  if_transmit = 0xffffffff80cb67a0 <if_transmit>, if_reassign = 0x0,
  if_get_counter = 0xffffffff80cade70 <if_get_counter_default>,
  if_requestencap = 0xffffffff80cb6940 <if_requestencap_default>,
  if_counters = {0xfffffe00004734e0, 0xfffffe00004734f0, 0xfffffe0000473500,
    0xfffffe0000473510, 0xfffffe0000473180, 0xfffffe0000473190,
    0xfffffe00004731a0, 0xfffffe00004731b0, 0xfffffe00004731c0,
    0xfffffe00004731d0, 0xfffffe00004731e0, 0xfffffe00004731f0},
  if_hw_tsomax = 65518, if_hw_tsomaxsegcount = 35, if_hw_tsomaxsegsize = 2048,
  if_snd_tag_alloc = 0x0, if_snd_tag_modify = 0x0, if_snd_tag_query = 0x0,
  if_snd_tag_free = 0x0, if_pcp = 255 '\377', if_netdump_methods = 0x0,
  if_epoch_ctx = {data = {0x0, 0x0}}, if_addr_et = {datap = {0x0,
      0xfffffe0018759ac0, 0xdeadbeef}, datai = {1}}, if_maddr_et = {datap = {
      0x0, 0x0, 0x0}, datai = {0}}, if_ispare = {0, 0, 0, 0}}

(kgdb) list tunclose
435      * tunclose - close the device - mark i/f down & delete
436      * routing info
437      */
438     static  int
439     tunclose(struct cdev *dev, int foo, int bar, struct thread *td)
440     {
441             struct tun_softc *tp;
442             struct ifnet *ifp;
443
444             tp = dev->si_drv1;
(kgdb)
445             ifp = TUN2IFP(tp);
446
447             mtx_lock(&tp->tun_mtx);
448             tp->tun_flags &= ~TUN_OPEN;
449             tp->tun_pid = 0;
450
451             /*
452              * junk all pending output
453              */
454             CURVNET_SET(ifp->if_vnet);
(kgdb)
455             IFQ_PURGE(&ifp->if_snd);
456
457             if (ifp->if_flags & IFF_UP) {
458                     mtx_unlock(&tp->tun_mtx);
459                     if_down(ifp);
460                     mtx_lock(&tp->tun_mtx);
461             }
462
463             /* Delete all addresses and routes which reference this interface. */
464             if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
(kgdb)
465                     struct ifaddr *ifa;
466
467                     ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
468                     mtx_unlock(&tp->tun_mtx);
469                     CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
470                             /* deal w/IPv4 PtP destination; unlocked read */
471                             if (ifa->ifa_addr->sa_family == AF_INET) {
472                                     rtinit(ifa, (int)RTM_DELETE,
473                                         tp->tun_flags & TUN_DSTADDR ? RTF_HOST : 0);
474                             } else {
(kgdb)
475                                     rtinit(ifa, (int)RTM_DELETE, 0);
476                             }
477                     }
478                     if_purgeaddrs(ifp);
479                     mtx_lock(&tp->tun_mtx);
480             }
481             if_link_state_change(ifp, LINK_STATE_DOWN);
482             CURVNET_RESTORE();
483
484             funsetown(&tp->tun_sigio);
(kgdb)
485             selwakeuppri(&tp->tun_rsel, PZERO + 1);
486             KNOTE_LOCKED(&tp->tun_rsel.si_note, 0);
487             TUNDEBUG (ifp, "closed\n");
488
489             cv_broadcast(&tp->tun_cv);
490             mtx_unlock(&tp->tun_mtx);
491             return (0);
492     }
493
494     static void
(kgdb)

Looking into rtinit()

[net/route.c]

void
rt_maskedcopy(struct sockaddr *src, struct sockaddr *dst, struct sockaddr *netmask)
{
        u_char *cp1 = (u_char *)src;
        u_char *cp2 = (u_char *)dst;
        u_char *cp3 = (u_char *)netmask;
        u_char *cplim = cp2 + *cp3;
        u_char *cplim2 = cp2 + *cp1;

        *cp2++ = *cp1++; *cp2++ = *cp1++; /* copies sa_len & sa_family */
        cp3 += 2;
        if (cplim > cplim2)
                cplim = cplim2;
        while (cp2 < cplim)
                *cp2++ = *cp1++ & *cp3++;
        if (cp2 < cplim2)
                bzero((caddr_t)cp2, (unsigned)(cplim2 - cp2));
}

This function copies the sockaddr data pointed by src to the buffer pointed by dst. More exactly, it performs bitwise AND operation on src and netmask, then copy the result to dst.

It is called from rtinit1() in net/route.c, which is in turn called by tunclose() in net/if_tun.c Line 472 or 475.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment