simple ansible traceraoute callback plugin used to run a traceroute whenever a node is "UNREACHABLE". NOTE: This callback plugin requires the pacakge `pytraceroute` to be installed.

traceroute.py

from __future__ import absolute_import
__metaclass__ = type

import socket
import time

from traceroute.core import Tracer

from ansible import constants as C
from ansible.plugins.callback import CallbackBase


DOCUMENTATION = '''
    callback: 'null'
    callback_type: stdout
    requirements:
      - set as main display callback
    short_description: Traceroute to `ansible_host` on failure.
    version_added: "2.7"
    description:
        - This callback plugin will run a traceroute whenever a node becomes "unreachable".
'''

class PyTracer(Tracer):
    def run(self):
        results = list()
        try:
            dst_ip = socket.gethostbyname(self.dst)
        except socket.error as e:
            raise IOError('Unable to resolve {}: {}', self.dst, e)

        text = 'traceroute to {} ({}), {} hops max'.format(
            self.dst,
            dst_ip,
            self.hops
        )

        results.append(text)

        while True:
            startTimer = time.time()
            receiver = self.create_receiver()
            sender = self.create_sender()
            sender.sendto(b'', (self.dst, self.port))

            addr = None
            try:
                data, addr = receiver.recvfrom(1024)
                entTimer = time.time()
            except socket.error:
                pass
            finally:
                receiver.close()
                sender.close()

            if addr:
                address = addr[0]
                timeCost = round((entTimer - startTimer) * 1000, 2)
                results.append('{:<4} {} {} ms'.format(self.ttl, address, timeCost))
                if address == dst_ip:
                    break
            else:
                print('{:<4} *'.format(self.ttl))

            self.ttl += 1

            if self.ttl > self.hops:
                break
        
        return results


class CallbackModule(CallbackBase):
    CALLBACK_VERSION = 2.0
    CALLBACK_TYPE = 'notification'
    CALLBACK_NAME = 'traceroute'
    CALLBACK_NEEDS_WHITELIST = False

    def _traceroute(self, host_address, hostname):
        tracer = PyTracer(
            dst=host_address,
            hops=int(32)
        )
        traced = tracer.run()
        self._display.display(
                "Traceroute to [{}] => {}".format(
                    hostname,
                    '\n'.join(traced)
                ),
                color=C.COLOR_ERROR,
                stderr=True
            )

    def v2_runner_on_unreachable(self, result, **kwargs):
        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if delegated_vars:
            self._traceroute(
                host_address=delegated_vars['ansible_host'],
                hostname=result._host.get_name()
            )
        else:
            self._traceroute(
                host_address=result._host.vars['ansible_host'],
                hostname=result._host.get_name()
            )

Example output

PLAY [Setup the utility location(s)] **************************************************************************************************************************************************************************************************

TASK [Gathering Facts] ****************************************************************************************************************************************************************************************************************
fatal: [test2]: UNREACHABLE! => {"changed": false, "msg": "Failed to connect to the host via ssh: Permission denied (publickey).", "unreachable": true}
Traceroute to [test2] => traceroute to ze12.openstack.org (104.130.124.187), 32 hops max
1    10.240.0.2 1.94 ms
2    72.4.118.226 1.65 ms
3    69.20.2.30 2.59 ms
4    69.20.2.112 2.38 ms
5    69.20.2.164 3.13 ms
6    10.25.1.95 31.77 ms
7    148.62.41.101 31.04 ms
8    148.62.41.123 32.38 ms
9    98.129.84.197 31.91 ms
10   104.130.124.187 32.54 ms
	to retry, use: --limit @/opt/openstack-ansible/playbooks/test.retry

PLAY RECAP ****************************************************************************************************************************************************************************************************************************
test2                      : ok=0    changed=0    unreachable=1    failed=0