clean.py

#!/usr/bin/env python3

from itertools import tee


def pairwise(iterable):
    """
    http://stackoverflow.com/a/11359174
    """
    a, b = tee(iterable)
    next(b, None)
    return zip(a, b)

with open('top-5K-cleaned.csv', 'w') as fw:
    with open('top-5K.csv','r') as fr:
        for line, next_line in pairwise(fr):
            if line != next_line:
                fw.write(line)