Sitemap Compare

compare-sitemap.py

"""
Quick and dirty script to compare the sitemap of an old nikola page with its hugo successor
"""
from lxml import etree
import requests


def get_urls(filename: str) -> set[str]:
    tree_sitemap = etree.parse(filename)
    ns = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"}
    urls = [el.text.replace("http://localhost:1313", "") 
            for el in tree_sitemap.xpath("//sitemap:urlset/sitemap:url/sitemap:loc", namespaces=ns)
            ]
    return set(urls)


old_urls = get_urls("sitemap_old.xml")
new_urls = get_urls("sitemap_new_en.xml").union(get_urls("sitemap_new_de.xml"))

def url_not_reachable(path: str) -> bool:
  return requests.get(f"http://localhost:1313/{path}").status_code >= 400

missing = list(old_urls - new_urls)
missing.sort()
not_reachable_missing = list(filter(url_not_reachable, missing))
print("Paths in the old page but not reachable on http://localhost:1313/:")
print("\n".join(not_reachable_missing))