FindDpSnippets.py

"""
Comparing entries in Snippets.txt by hashing the snippets
I made a duplicate to text it.
Hash the snippets then check set() s of the HASH for duplicates.
Just playing with Python Works great. Snippets.txt is in my gists
it is loaded with snippets and script.
"""
import hashlib
import string
Hash = []
hash2 = set()

def hashIt(mystring):
    hash_object = hashlib.md5(mystring.encode())
    return(hash_object.hexdigest())

def findDup(Hash):
    seen = set()
    Hash
    count = 0
    HashIn = str(Hash)
    h2 = HashIn.replace("\\n", "\n")
    h2 = h2.replace("', '", "");h2 = h2.replace("'", "")
    h2 = h2.replace("[", "");h2 = h2.replace("]", "")
    h2 = h2.split("\n")
    for h in h2:
        count = count + 1
        if h in seen:print count," - This is a duplicate :",h 
        else:seen.add(h)
            
# print "Adding Count :",count," ",h,"\n"    


f = open("Snippets.txt", "r")
snippets = f.read().split("-------new entry -------")
for snippet in snippets:
    printable = set(string.printable)
    snippet = filter(lambda x: x in printable, snippet)
    hashin = hashIt(snippet)
    Hash.extend(hashin+"\n")
    hash2.update(snippet,)
    #print hashIt(snippet),"\n",snippet
    findDup(Hash)