Last active
July 12, 2023 08:34
-
-
Save ZaphodB/4417b321ea68a7fb096b6522b9bd38df to your computer and use it in GitHub Desktop.
This takes https://www.blocklist.de/en/export.html IPs and puts them in ipsets that can be used for firewalling with netfilter/iptables
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[Unit] | |
Description=update ipsets from blocklist.de | |
Wants=network-online.target | |
After=network-online.target | |
ConditionACPower=true | |
[Service] | |
Type=oneshot | |
# Lower CPU and I/O priority. | |
Nice=19 | |
CPUSchedulingPolicy=batch | |
IOSchedulingClass=best-effort | |
IOSchedulingPriority=7 | |
IOWeight=100 | |
Restart=no | |
LogRateLimitIntervalSec=0 | |
ExecStart=systemd-inhibit --who="ipset" --why="Prevent interrupting our ipset update" /etc/ipset-delta.py |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[Unit] | |
Description=Run blocklist ipset update | |
[Timer] | |
OnBootSec=30min | |
OnUnitActiveSec=30min | |
Persistent=true | |
[Install] | |
WantedBy=timers.target |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import shutil | |
import requests | |
import os | |
import ipaddress | |
from pathlib import Path | |
oldFileName = '/etc/blocklist.all.txt' | |
newFileName = '/etc/blocklist.all.txt.new' | |
try: | |
shutil.move(newFileName, oldFileName) | |
except: | |
pass | |
url = 'https://lists.blocklist.de/lists/all.txt' | |
filename = Path(newFileName) | |
filename.touch(exist_ok=True) | |
#response = requests.get(url, headers={'User-Agent': 'Mozilla'}) | |
with requests.get(url, headers={'User-Agent': 'Mozilla'}) as r: | |
r.encoding = r.apparent_encoding | |
with open(filename, 'wb') as f: | |
f.write(r.content) | |
f.close() | |
oldFile = open(oldFileName, 'r') | |
newFile = open(newFileName, 'r') | |
oldSet = set() | |
for line in oldFile: | |
try: | |
ipaddress.ip_address(line.rstrip()) | |
except: | |
continue | |
oldSet.add(line.rstrip()) | |
oldFile.close() | |
newSet = set() | |
for line in newFile: | |
try: | |
ipaddress.ip_address(line.rstrip()) | |
except: | |
continue | |
newSet.add(line.rstrip()) | |
newFile.close() | |
unchanged = oldSet & newSet | |
todelete = oldSet - newSet | |
additions = newSet - oldSet | |
for item in todelete: | |
print("removing: \"%s\"" % item) | |
try: | |
ipaddress.IPv6Address(item) | |
os.system("/usr/sbin/ipset del blocklist6 %s" % item) | |
except: | |
os.system("/usr/sbin/ipset del blocklist %s" % item) | |
for item in additions: | |
print("need to add: \"%s\"" % item) | |
try: | |
ipaddress.IPv6Address(item) | |
os.system("/usr/sbin/ipset add blocklist6 %s" % item) | |
except: | |
os.system("/usr/sbin/ipset add blocklist %s" % item) | |
os.system("ipset save > /etc/ipsets.conf") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import shutil | |
import requests | |
import os | |
import subprocess | |
from io import StringIO | |
import ipaddress | |
result = subprocess.run(['/usr/sbin/ipset', 'list', 'blocklist'], stdout=subprocess.PIPE, check=True) | |
oldSet = set() | |
buf=StringIO(result.stdout.decode('utf-8')) | |
for line in buf.read().split("\n"): | |
try: | |
ipaddress.IPv4Address(line.rstrip()) | |
except: | |
continue | |
#print("line: \"%s\"" % line) | |
oldSet.add(line.rstrip()) | |
newFileName = '/etc/blocklist.all.txt' | |
newFile = open(newFileName, 'r') | |
newSet = set() | |
for line in newFile: | |
try: | |
ipaddress.IPv4Address(line.rstrip()) | |
except: | |
continue | |
newSet.add(line.rstrip()) | |
newFile.close() | |
unchanged = oldSet & newSet | |
todelete = oldSet - newSet | |
additions = newSet - oldSet | |
for item in todelete: | |
print("removing: \"%s\"" % item) | |
os.system("/usr/sbin/ipset del blocklist %s" % item) | |
for item in additions: | |
print("adding: \"%s\"" % item) | |
os.system("/usr/sbin/ipset add blocklist %s" % item) | |
os.system("ipset save > /etc/ipsets.conf") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/python3 | |
import shutil | |
import requests | |
import os | |
import subprocess | |
from io import StringIO | |
import ipaddress | |
result = subprocess.run(['/usr/sbin/ipset', 'list', 'blocklist6'], stdout=subprocess.PIPE, check=True) | |
oldSet = set() | |
buf=StringIO(result.stdout.decode('utf-8')) | |
for line in buf.read().split("\n"): | |
try: | |
ipaddress.IPv6Address(line.rstrip()) | |
except: | |
continue | |
#print("line: \"%s\"" % line) | |
oldSet.add(line.rstrip()) | |
newFileName = '/etc/blocklist.all.txt' | |
newFile = open(newFileName, 'r') | |
newSet = set() | |
for line in newFile: | |
try: | |
ipaddress.IPv6Address(line.rstrip()) | |
except: | |
continue | |
newSet.add(line.rstrip()) | |
newFile.close() | |
unchanged = oldSet & newSet | |
todelete = oldSet - newSet | |
additions = newSet - oldSet | |
for item in todelete: | |
print("removing: \"%s\"" % item) | |
os.system("/usr/sbin/ipset del blocklist6 %s" % item) | |
for item in additions: | |
print("adding: \"%s\"" % item) | |
os.system("/usr/sbin/ipset add blocklist6 %s" % item) | |
os.system("ipset save > /etc/ipsets.conf") |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[Unit] | |
Description=ipset persistancy service | |
DefaultDependencies=no | |
Requires=netfilter-persistent.service | |
Before=network.target | |
Before=netfilter-persistent.service | |
ConditionFileNotEmpty=/etc/ipsets.conf | |
[Service] | |
Type=oneshot | |
RemainAfterExit=yes | |
ExecStart=/usr/sbin/ipset restore -f -! /etc/ipsets.conf | |
# save on service stop, system shutdown etc. | |
ExecStop=/usr/sbin/ipset save blacklist -f /etc/ipsets.conf | |
[Install] | |
WantedBy=multi-user.target | |
RequiredBy=netfilter-persistent.service |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment