Removes duplicate citations in bib files

remove_duplicate.py

import re

with open("bibliography.bib", "r", encoding="utf-8") as f:
    lines = f.readlines()

entries = []
current = []
for line in lines:
    if line.lstrip().startswith("@") and current:
        entries.append("".join(current))
        current = []
    current.append(line)
if current:
    entries.append("".join(current))

seen_titles = set()
unique_entries = []

for entry in entries:
    match = re.search(r'\btitle\b\s*=\s*\{(.*?)\}', entry, re.IGNORECASE | re.DOTALL)
    if match:
        title = match.group(1)
        norm_title = " ".join(title.split())
        if norm_title in seen_titles:
            continue
        seen_titles.add(norm_title)
    unique_entries.append(entry)

with open("merged.bib", "w", encoding="utf-8") as f:
    f.write("\n".join(unique_entries))