Fetch all IP ranges (CIDR) for major cloud providers (AWS, GCP, Azure, Cloudflare) to use for bot or crawler detection

fetch_ips.py

import requests, json, ipaddress

def get_cidr_ranges(provider):
    if provider == 'aws':
        url = 'https://ip-ranges.amazonaws.com/ip-ranges.json'
        base_key = 'prefixes'
        key = 'ip_prefix'
    elif provider == 'gcp':
        url = 'https://www.gstatic.com/ipranges/cloud.json'
        base_key = 'prefixes'
        key = 'ipv4Prefix'
    elif provider == 'github':
        url = 'https://api.github.com/meta'
        base_key = 'actions'
        key = None
    elif provider == 'cloudflare':
        url = 'https://www.cloudflare.com/ips-v4'
        base_key = None
        key = None
    elif provider == 'azure':
        url = 'https://download.microsoft.com/download/7/1/D/71D86715-5596-4529-9B13-DA13A5DE5B63/ServiceTags_Public_20230522.json'
        base_key = None
        key = None
    

    if base_key is not None:
        data = requests.get(url).json()
        if key is not None:
            return [x[key] for x in data[base_key] if key in x]
        else:
            return [x for x in data[base_key] if ":" not in x]

    elif provider == 'cloudflare':
        return [line.strip() for line in requests.get(url).text.splitlines()]

    elif provider == 'azure':
        data = requests.get(url).json()
        ipv4_list = []
        for region in data["values"]:
            ipv4_list.extend([x for x in region["properties"]["addressPrefixes"] if ":" not in x])
        return list(set(ipv4_list))

# Write all to file for later use
with open("ipv4-cidrs.txt", "w") as f:
	f.write(",".join(get_cidr_ranges('aws')))
	f.write(",".join(get_cidr_ranges('gcp')))
	f.write(",".join(get_cidr_ranges('cloudflare')))
	f.write(",".join(get_cidr_ranges('azure')))


# Alternatively get all the CIDRs and check if our IP matches any CIDR
cidr_list = []
cidr_list.extend(get_cidr_ranges('aws'))
cidr_list.extend(get_cidr_ranges('gcp'))
cidr_list.extend(get_cidr_ranges('cloudflare'))
cidr_list.extend(get_cidr_ranges('azure'))

my_ip = '3.2.34.1' # AWS
is_bot = any([ipaddress.ip_address(my_ip) in ipaddress.ip_network(cidr) for cidr in cidr_list])

print(f"Is {my_ip} a bot? {is_bot}")