Script to find duplicate notes in an Anki collection. Some special-casing for certain criteria.

duplicates-finder.py

from codecs import open
from collections import defaultdict
import re
"""
To use, export decks so that
Core2k: core2k.txt
Core6k: core6k.txt
Vocabulary (including both user-generated; CorePlus; and Core2-6K): out.txt

Before export, tag non-leech suspended notes with "rololo"

Toggle comments to compare reading
"""
core=[line for line in open(r'C:\Users\Owner\Desktop\core2k.txt', encoding='utf8')  if 'rololo' not in line] + [line for line in open(r'C:\Users\Owner\Desktop\core6k.txt', encoding='utf8') if 'rololo' not in line]
plus=[line for line in open(r'C:\Users\Owner\Desktop\coreplus.txt', encoding='utf8')  if 'rololo' not in line]
others = [line for line in open(r'C:\Users\Owner\Desktop\out.txt', encoding='utf8') if line not in plus and line not in core and 'rololo' not in line]

dupes=defaultdict(list)
KANA_RANGE=u'([\u30a0-\u30ff]|[\u3040-\u309f])'

for line in plus:
  e,r,m=line.split('\t')[:3]
##	dupes[''.join(re.findall(KANA_RANGE, r))].append((e,m, 'plus'))
	dupes[e].append((r,m, 'plus'))
	
for line in core:
	e,r,m=line.split('\t')[:3]
##	dupes[''.join(re.findall(KANA_RANGE, r))].append((e,m, 'core'))
	dupes[e].append((r,m, 'core'))

for line in others:
        e, m, r = line.split('\t')[:3]
##        dupes[''.join(re.findall(KANA_RANGE, r))].append((e,m, 'other'))
        dupes[e].append((r,m, 'other'))

dupes={e: vals for e, vals in dupes.items() if len(vals)>1}
dupes={e: vals for e, vals in dupes.items() if len({v[2] for v in vals})>1}

for e, vals in dupes.items():
	print e
	for reading, meaning, tag in vals:
		print '\t', reading, '\t', meaning, '\t', tag