Skip to content

Instantly share code, notes, and snippets.

@ZaphodB
Last active July 12, 2023 08:34
Show Gist options
  • Save ZaphodB/4417b321ea68a7fb096b6522b9bd38df to your computer and use it in GitHub Desktop.
Save ZaphodB/4417b321ea68a7fb096b6522b9bd38df to your computer and use it in GitHub Desktop.
This takes https://www.blocklist.de/en/export.html IPs and puts them in ipsets that can be used for firewalling with netfilter/iptables
[Unit]
Description=update ipsets from blocklist.de
Wants=network-online.target
After=network-online.target
ConditionACPower=true
[Service]
Type=oneshot
# Lower CPU and I/O priority.
Nice=19
CPUSchedulingPolicy=batch
IOSchedulingClass=best-effort
IOSchedulingPriority=7
IOWeight=100
Restart=no
LogRateLimitIntervalSec=0
ExecStart=systemd-inhibit --who="ipset" --why="Prevent interrupting our ipset update" /etc/ipset-delta.py
[Unit]
Description=Run blocklist ipset update
[Timer]
OnBootSec=30min
OnUnitActiveSec=30min
Persistent=true
[Install]
WantedBy=timers.target
#!/usr/bin/python3
import shutil
import requests
import os
import ipaddress
from pathlib import Path
oldFileName = '/etc/blocklist.all.txt'
newFileName = '/etc/blocklist.all.txt.new'
try:
shutil.move(newFileName, oldFileName)
except:
pass
url = 'https://lists.blocklist.de/lists/all.txt'
filename = Path(newFileName)
filename.touch(exist_ok=True)
#response = requests.get(url, headers={'User-Agent': 'Mozilla'})
with requests.get(url, headers={'User-Agent': 'Mozilla'}) as r:
r.encoding = r.apparent_encoding
with open(filename, 'wb') as f:
f.write(r.content)
f.close()
oldFile = open(oldFileName, 'r')
newFile = open(newFileName, 'r')
oldSet = set()
for line in oldFile:
try:
ipaddress.ip_address(line.rstrip())
except:
continue
oldSet.add(line.rstrip())
oldFile.close()
newSet = set()
for line in newFile:
try:
ipaddress.ip_address(line.rstrip())
except:
continue
newSet.add(line.rstrip())
newFile.close()
unchanged = oldSet & newSet
todelete = oldSet - newSet
additions = newSet - oldSet
for item in todelete:
print("removing: \"%s\"" % item)
try:
ipaddress.IPv6Address(item)
os.system("/usr/sbin/ipset del blocklist6 %s" % item)
except:
os.system("/usr/sbin/ipset del blocklist %s" % item)
for item in additions:
print("need to add: \"%s\"" % item)
try:
ipaddress.IPv6Address(item)
os.system("/usr/sbin/ipset add blocklist6 %s" % item)
except:
os.system("/usr/sbin/ipset add blocklist %s" % item)
os.system("ipset save > /etc/ipsets.conf")
#!/usr/bin/python3
import shutil
import requests
import os
import subprocess
from io import StringIO
import ipaddress
result = subprocess.run(['/usr/sbin/ipset', 'list', 'blocklist'], stdout=subprocess.PIPE, check=True)
oldSet = set()
buf=StringIO(result.stdout.decode('utf-8'))
for line in buf.read().split("\n"):
try:
ipaddress.IPv4Address(line.rstrip())
except:
continue
#print("line: \"%s\"" % line)
oldSet.add(line.rstrip())
newFileName = '/etc/blocklist.all.txt'
newFile = open(newFileName, 'r')
newSet = set()
for line in newFile:
try:
ipaddress.IPv4Address(line.rstrip())
except:
continue
newSet.add(line.rstrip())
newFile.close()
unchanged = oldSet & newSet
todelete = oldSet - newSet
additions = newSet - oldSet
for item in todelete:
print("removing: \"%s\"" % item)
os.system("/usr/sbin/ipset del blocklist %s" % item)
for item in additions:
print("adding: \"%s\"" % item)
os.system("/usr/sbin/ipset add blocklist %s" % item)
os.system("ipset save > /etc/ipsets.conf")
#!/usr/bin/python3
import shutil
import requests
import os
import subprocess
from io import StringIO
import ipaddress
result = subprocess.run(['/usr/sbin/ipset', 'list', 'blocklist6'], stdout=subprocess.PIPE, check=True)
oldSet = set()
buf=StringIO(result.stdout.decode('utf-8'))
for line in buf.read().split("\n"):
try:
ipaddress.IPv6Address(line.rstrip())
except:
continue
#print("line: \"%s\"" % line)
oldSet.add(line.rstrip())
newFileName = '/etc/blocklist.all.txt'
newFile = open(newFileName, 'r')
newSet = set()
for line in newFile:
try:
ipaddress.IPv6Address(line.rstrip())
except:
continue
newSet.add(line.rstrip())
newFile.close()
unchanged = oldSet & newSet
todelete = oldSet - newSet
additions = newSet - oldSet
for item in todelete:
print("removing: \"%s\"" % item)
os.system("/usr/sbin/ipset del blocklist6 %s" % item)
for item in additions:
print("adding: \"%s\"" % item)
os.system("/usr/sbin/ipset add blocklist6 %s" % item)
os.system("ipset save > /etc/ipsets.conf")
[Unit]
Description=ipset persistancy service
DefaultDependencies=no
Requires=netfilter-persistent.service
Before=network.target
Before=netfilter-persistent.service
ConditionFileNotEmpty=/etc/ipsets.conf
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/sbin/ipset restore -f -! /etc/ipsets.conf
# save on service stop, system shutdown etc.
ExecStop=/usr/sbin/ipset save blacklist -f /etc/ipsets.conf
[Install]
WantedBy=multi-user.target
RequiredBy=netfilter-persistent.service
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment