Get the most up-to-date list of IP addresses for crawler bots, belonging to Google and Bing.

get_bot_ip_addresses.py

import ipaddress

import requests
import pandas as pd

def bot_ip_addresses():
    bots_urls = {
        'google': 'https://developers.google.com/search/apis/ipranges/googlebot.json',
        'bing': 'https://www.bing.com/toolbox/bingbot.json'
    }
    ip_addresses = []
    for bot, url in bots_urls.items():
        bot_resp = requests.get(url)
        for iprange in bot_resp.json()['prefixes']:
            network = iprange.get('ipv4Prefix')
            if network:
                ip_list = [(bot, str(ip)) for ip in ipaddress.IPv4Network(network)]
                ip_addresses.extend(ip_list)
    return pd.DataFrame(ip_addresses, columns=['bot_name', 'ip_address'])

Might also find this repo of use https://github.com/AnTheMaker/GoodBots

@johnmurch
Interesting. Keeping and updating a static list of IPs can be another useful approach.
Thanks for sharing!