Scrape OpnSense DHCP Leases Status Page and Export Results as List

scrape_opnsense_dhcp_leases.py

#!/usr/bin/env python3

#
# This python script provides a function to query the opnSense (v21.3 - v23.1.x) dhcp leases status page and 
# return a list of tuples including ip, hostname, and mac address. This script will not work with version 23.7.x+ 
# due to changes in the DHCP lease page.
# See comment from dheldt on modified version that works using api searchLeases page. 
# To use the original script, ensure LXML is installed via package manager or via pip. 
#
# 27-Mar-2021 - Original release
# 17-Jul-2022 - Fix url in scrape function. Add error trapping for case where user/pass are not set up correctly.
# 24-Sep-2023 - Add note about 23.7.x incompatibility.
#


import sys
import requests
from lxml import html
import re

url  = "http://192.168.1.1/status_dhcp_leases.php" #change url to match your opnSense machine address. Note http or https!
user = 'your_username'  #Username for opnSense login (default is 'root')
password = 'your_password' #Password for opnSense login

def scrape_opnsense_dhcp(url, user, password):
   ip = []
   mac = []
   hostname = []

   s = requests.session()
   r = s.get(url,verify = False)

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))

   payload = {
'login' : 'Login',
'usernamefld' : user,
'passwordfld' : password
}

   r = s.post(url,data=payload,verify = False,headers={"X-CSRFToken":csrf.group(1)})
   r = s.get(url,verify = False)
   tree = html.fromstring(r.content)
   tr_elements = tree.xpath('//tr')
   try:
      headers = [header.text for header in tr_elements[0]]
   except IndexError:
      print("Error retrieving lease list. Are you sure username and password were set up in script?")
      quit()

   ip.extend(tree.xpath('//table[@class="table table-striped"]//tbody//tr//td[' + str(headers.index('IP address') + 1) +']//text()'))

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('MAC address') + 1) +']'):
      if bool(re.search(r'([0-9a-f]{2}(?::[0-9a-f]{2}){5})', node.text)):
         mac.append(node.text)

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Hostname') + 1) +']'):
      if node.text is None:
         hostname.append('no_hostname')
      else:
         hostname.append(node.text)

   for i in range(len(mac)):
      mac[i] = mac[i].strip()

   return(list(zip(ip, mac, hostname)))

if __name__ == "__main__":     
    dhcp_list = scrape_opnsense_dhcp(url, user, password)

    for entry in dhcp_list:
            print(entry)

Works perfect! Thank you for taking time to update it. I used it with https and no issues besides the cert warning.

This is awesome. Thank you!

I updated the code to work with latest OPNSense version 23.1.9. It seems that it required an additional field in POST request.
Also I included the description of each device from DHCP overview.

Usage:
OPNSENSE_USERNAME=root OPNSENSE_PASSWORD='abc' python3 opnsense-dhcp-ip-overview.py

#!/usr/bin/env python3

#
# # This python script provides a function to query the opnSense (+v21.3) dhcp leases status page and return a list of tuples including 
# ip, hostname, and mac address. To use, ensure LXML is installed via package manager or via pip.
#
# 27-Mar-2021 - Original release
# 17-Jul-2022 - Fix url in scrape function. Add error trapping for case where user/pass are not set up correctly.
# 13-Jun-2023 - Fix missing field in POST request for new OPNSense version. Added field Description.
#

# Use:
# OPNSENSE_USERNAME=marcel OPNSENSE_PASSWORD='abc' python3 opnsense-dhcp-ip-overview.py

import os
import sys
import requests
import urllib3
import re
from lxml import html

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

url  = "https://192.168.1.1/status_dhcp_leases.php" #change url to match your opnSense machine address. Note http or https!
user = os.getenv('OPNSENSE_USERNAME')
password = os.getenv('OPNSENSE_PASSWORD')

def scrape_opnsense_dhcp(url, user, password):
   ip = []
   mac = []
   hostname = []
   description = []

   s = requests.session()
   r = s.get(url,verify = False)

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))
   csrf_token = csrf.group(1)
   
   matchme = 'input type="hidden" name="([^"]*)" value="([^"]*)"'
   hidden_field = re.search(matchme,str(r.text))
   hidden_field_id = hidden_field.group(1)
   hidden_field_value = hidden_field.group(2)

   payload = {
'login' : '1',
'usernamefld' : user,
'passwordfld' : password
}

   payload[hidden_field_id] = hidden_field_value

   r = s.post(url,data=payload,verify = False,headers={"X-CSRFToken":csrf_token})
   r = s.get(url,verify = False)
   tree = html.fromstring(r.content)
   tr_elements = tree.xpath('//tr')
   try:
      headers = [header.text for header in tr_elements[0]]
   except IndexError:
      print("Error retrieving lease list. Are you sure username and password were set up in script?")
      quit()

   ip.extend(tree.xpath('//table[@class="table table-striped"]//tbody//tr//td[' + str(headers.index('IP address') + 1) +']//text()'))

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('MAC address') + 1) +']'):
      if bool(re.search(r'([0-9a-f]{2}(?::[0-9a-f]{2}){5})', node.text)):
         mac.append(node.text)

   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Hostname') + 1) +']'):
      if node.text is None:
         hostname.append('no_hostname')
      else:
         hostname.append(node.text)
         
   for node in tree.xpath('//table[@class="table table-striped"]//tbody//tr//td['+ str(headers.index('Description') + 1) +']'):
      if node.text is None:
         description.append('no_description')
      else:
         description.append(node.text)

   for i in range(len(mac)):
      mac[i] = mac[i].strip()

   return(list(zip(ip, mac, hostname, description)))

if __name__ == "__main__":     
    dhcp_list = scrape_opnsense_dhcp(url, user, password)

    for entry in dhcp_list:
            print(entry)

Hi there,

for me, neither version works (having an OPNsense 23.7.3-amd64 running).
First of all because on my system there is no status_dhcp_leases.php present. Instead I get a "Page not found"-Error.

But I stumplet across /api/dhcpv4/leases/searchlease, which does work on my system. A simple get Request there (in a browser where I am logged in) gives a json object of the form:

{"total":18,"rowCount":18,"current":1,"rows":[{"address":"192.168.x.y","type":"static","mac":"SomeMac","starts":"","ends":"","hostname":"firstHost","descr":"","if_descr":"default","if":"opt8","state":"active","status":"offline","man":"manufacturer"},
....

]

so ... maybe this is the right way to go and fix instead of parsing html?

And, thank you a lot for your work and script, which has led me to the get-endpoint in the first place!

Yes, this is broken as of 23.7.x due to changes in the lease page. I haven't dug into trying to fix yet.

I wasn't aware of the searchlease endpoint but it does indeed seem like an opportunity to simplify. Will take a look when I get a little time in the next few days.

If you adept as follows, it does (currently) work:

mainUrl = "https://192.168.1.1/"
url  = "https://192.168.1.1/api/dhcpv4/leases/searchlease

   s = requests.session()
   r = s.get(mainUrl,verify = False)

   if not(r == 200):
   	print("failed to connect")

   matchme = '"X-CSRFToken", "(.*)" \);'
   csrf = re.search(matchme,str(r.text))
   csrf_token = csrf.group(1)
   
   matchme = 'input type="hidden" name="([^"]*)" value="([^"]*)"'
   hidden_field = re.search(matchme,str(r.text))
   hidden_field_id = hidden_field.group(1)
   hidden_field_value = hidden_field.group(2)

   payload = {
'login' : '1',
'usernamefld' : user,
'passwordfld' : password
}

   payload[hidden_field_id] = hidden_field_value
   r = s.post(mainUrl,data=payload,verify = False,headers={"X-CSRFToken":csrf_token})
  
   if not(r == 200):
   	print("failed to log in")
   	
   r = s.get(url,verify = False)
   
   return r.json()

Note that you need to use two different urls now, because the log in does not work for the rest endpoint.

regards,

daniel

Thank you for working through this!

It turns out it is simpler to set up an api access key as described here:
https://docs.opnsense.org/development/how-tos/api.html.

The code to retrieve the leases can then simply be reduced to a single GET request.

import json
import requests

# define endpoint and credentials
api_key = '---------'   #use your unique key
api_secret = '-----------------'  #use your unique secret
url = 'http://opnsense.home/api/dhcpv4/leases/searchlease'
r = requests.get(url,
                 verify='OPNsense.pem',
                 auth=(api_key, api_secret))

if r.status_code == 200:
    return r.json()
else:
    print ('Connection / Authentication issue, response received:')