hosts file updater for pihole

gistfile1.txt

#!/usr/bin/env python3.6

'''
Pihole is great, but the admin interface only displays device details 
by IP address which can be confusing. This script changes the display
from IP address to a more recognizable hostname. And as a bonus, attaches
the profile (from fingerbank.org) of the device to the hostname as well - 
so instead of something like 192.168.1.101, you see galaxys6-samsung. 
Shweet. 
Usage notes
- sudo python3.6 discovery.py
- Tested with python 3.6 only
- Requires fingerbank API key (https://api.fingerbank.org/users/register) in a secrets.py file.
- Displays log messages at appropriate times
License: MIT.
'''

import os
from scapy.all import *
from python_hosts import Hosts, HostsEntry
from shutil import copyfile
import sys
import urllib3
import requests
import json
import secrets


'''
Global stuff
'''

 
interface = "wlan0"
fingerbank_url = 'https://api.fingerbank.org/api/v2/combinations/interrogate'
headers = {
    'Content-Type': 'application/json',
}

params = (
    ('key', secrets.API_KEY),
)

'''
Log message for troubleshooting
'''

def log_fingerbank_error(e, response):
    print(f' HTTP error: {e}')
    responses = {
        404: "No device was found the the specified combination",
        502: "No API backend was able to process the request.",
        429: "The amount of requests per minute has been exceeded.",
        403: "This request is forbidden. Your account may have been blocked.",
        401: "This request is unauthorized. Either your key is invalid or wasn't specified."
    }
    print(responses.get(response.status_code, "Fingerbank API returned some unknown error"))
    return

def log_packet_info(packet):
    #print(packet.summary())
    #print(ls(packet))
    print('---')
    types = {
        1: "New DHCP Discover",
        2: "New DHCP Offer",
        3: "New DHCP Request",
        5: "New DHCP Ack",
        8: "New DHCP Inform"
    }
    if DHCP in packet:
        print(types.get(packet[DHCP].options[0][1], "Some Other DHCP Packet"))
    return

def log_fingerbank_response(json_response):
    #print(json.dumps(json_response, indent=4))
    print(f"Device Profile: {json_response['device']['name']}, Confidence score: {json_response['score']}")

# https://jcutrer.com/howto/dev/python/python-scapy-dhcp-packets
def get_option(dhcp_options, key):
    must_decode = ['hostname', 'domain', 'vendor_class_id']
    try:
        for i in dhcp_options:
            if i[0] == key:
                # If DHCP Server Returned multiple name servers 
                # return all as comma seperated string.
                if key == 'name_server' and len(i) > 2:
                    return ",".join(i[1:])
                # domain and hostname are binary strings,
                # decode to unicode string before returning
                elif key in must_decode:
                    return i[1].decode()
                else: 
                    return i[1]        
    except:
        pass

def handle_dhcp_packet(packet):
    log_packet_info(packet)
    if DHCP in packet:
        requested_addr = get_option(packet[DHCP].options, 'requested_addr')
        hostname = get_option(packet[DHCP].options, 'hostname')
        param_req_list = get_option(packet[DHCP].options, 'param_req_list')
        vendor_class_id = get_option(packet[DHCP].options, 'vendor_class_id')
        print(f"Host {hostname} ({packet[Ether].src}) requested {requested_addr}.")
        device_profile = profile_device(param_req_list, packet[Ether].src, vendor_class_id)
        if ((device_profile != -1) and requested_addr):
            update_hosts_file(requested_addr, hostname, device_profile)
    return

def profile_device(dhcp_fingerprint, macaddr, vendor_class_id):
    data = {}
    data['dhcp_fingerprint'] = ','.join(map(str, dhcp_fingerprint))
    data['debug'] = 'on'
    data['mac'] = macaddr
    data['vendor_class_id'] = vendor_class_id
    print(f"Will attempt to profile using {dhcp_fingerprint}, {macaddr}, and {vendor_class_id}")

    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

    try:
        response = requests.post(fingerbank_url, 
        headers=headers, 
        params=params, 
        data=json.dumps(data))
    except requests.exceptions.HTTPError as e:
        log_fingerbank_error(e, response)
        return -1
 
    log_fingerbank_response(response.json())

     # If score is less than 30, there is very little confidence on the returned profile. Ignore it.
    if (response.json()['score'] < 30):
        return -1
    
    return response.json()['device']['name']

'''
Update the hosts file with <hostname>-<profile> for hostname
'''

def update_hosts_file(address,hostname,profile):
    if profile is not None:
        copyfile("/etc/hosts", "hosts")
        etchostname = profile.replace(" ", "_") + ("-" + hostname if hostname else "")
        print(f"Updating hostname as: {etchostname} with {address}")

        hosts = Hosts(path='hosts')
        hosts.remove_all_matching(name=etchostname)
        new_entry = HostsEntry(entry_type='ipv4', address=address, names=[etchostname])
        hosts.add([new_entry])
        hosts.write()
        copyfile("hosts", "/etc/hosts")

        print(f"Updated Host name for hostsfile is {etchostname}")

            
print("Starting\n")
sniff(iface = interface, filter='udp and (port 67 or 68)', prn = handle_dhcp_packet, store = 0)
print("\n Shutting down...")

'''
End of file
'''

I can't get this to work. Can you provide how you installed 3.6 and scapy so I can try to replicate.

I can't get this to work. Can you provide how you installed 3.6 and scapy so I can try to replicate.

Hi. I just did this.

Try this one-liner to install python 3.6

sudo apt-get update -y && sudo apt-get install build-essential tk-dev libncurses5-dev libncursesw5-dev libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev libffi-dev -y && wget https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tar.xz && tar xf Python-3.6.0.tar.xz && cd Python-3.6.0 && ./configure && make -j 4 && sudo make altinstall && cd .. && sudo rm -r Python-3.6.0 && rm Python-3.6.0.tar.xz && sudo apt-get --purge remove build-essential tk-dev libncurses5-dev libncursesw5-dev libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev libffi-dev -y && sudo apt-get autoremove -y && sudo apt-get clean

If it works you can verify with

python3.6 -V

Install some required packages:

sudo pip3.6 install pip --upgrade
sudo pip3.6 install python-hosts
sudo pip3.6 install urllib3
sudo pip3.6 install requests

also need tcpdump

sudo apt-get install tcpdump

and then we need scapy

git clone https://github.com/secdev/scapy
cd scapy
sudo python3.6 setup.py install

Now this should work fine - but this pihole-hosts-building-script only works when I run it from inside the scapy folder..

Also it crashes after it's found one host with exit message

New DHCP Ack
Host None (macaddress removed) requested None.
Traceback (most recent call last):
  File "script.py", line 164, in <module>
    sniff(iface = interface, filter='udp and (port 67 or 68)', prn = handle_dhcp_packet, store = 0)
  File "/home/pi/scapy/scapy/sendrecv.py", line 918, in sniff
    r = prn(p)
  File "script.py", line 111, in handle_dhcp_packet
    device_profile = profile_device(param_req_list, packet[Ether].src, vendor_class_id)
  File "script.py", line 118, in profile_device
    data['dhcp_fingerprint'] = ','.join(map(str, dhcp_fingerprint))
TypeError: 'NoneType' object is not iterable

So i'm not sure what is up now..

I am observing the same error as you. I wrote a little bash script that cron wakes up every couple of minutes and checks to see if the process is still running (uses ps) and if it's not, just restarts it. Hopefully, at some point, someone will figure out why it fails. My copy works fine in the /home/pi directory.

But thanks for this routine, it's cool.

@eresgit you are a hero, excellent install instructions

I had to make a slight change as I am running on ethernet (eth0) rather than wireless (wlan0) on my pi but it was not called eth0, it was changed to something different on my pi-hole setup. Found it with an ifconfig and changed the code at line #34

If anyone wonders about secret for the Fingerbank API, just create a secret.py in same folder as the script and add the line
API_KEY="<your api key>"

To help others out wanting to use this script, the code from @eresgit works for me if you change every 3.6.0 to 3.6.9, and it doesn't need discovery.py to be inside the scapy folder for me

Also, if you're pretty new to linux like me, to get it to run automatically you make a file like devicename.sh (I put it in the home folder) with the following in it:

#!/bin/bash
sudo python3.6 /home/pi/discovery.py

then use

chmod u+x devicename.sh
sudo crontab -e

and at the bottom of that file write @reboot /home/pi/devicename.sh