stdavis · September 15, 2017 16:22
diff --git a/find_duplicates.py b/find_duplicates.py
 '''
 find_duplicates.py

 A module that contains code that finds duplicate features and notifies the owner of them.
 '''

 from os.path import basename

 import arcpy
 import messaging
 from xxhash import xxh64

 hashes = {}


 def is_skip_field(field):
    return 'SHAPE' in field.upper() or field.upper() in ['GLOBAL_ID', 'GLOBALID'] or field.startswith('OBJECTID')


 def get_fields(dataset):
    describe = arcpy.Describe(dataset)
    fields = [field.name for field in describe.fields if not is_skip_field(field.name)]

    if describe.datasetType.lower() != 'table':
        fields.append('SHAPE@WKT')
    fields.append('OID@')

    return fields


 def main(dataset, owner_email):
    print('looking for duplicates')
    with arcpy.da.SearchCursor(dataset, get_fields(dataset)) as cursor:
        for row in cursor:
            row_hash = xxh64(str(row[:-1])).hexdigest()
            group = hashes.setdefault(row_hash, [])
            group.append(row[-1])

    duplicate_groups = [group for group in hashes.values() if len(group) > 1]

    if len(duplicate_groups) > 0:
        dup_ids = []
        for group in duplicate_groups:
            dup_ids += group[1:]
        emailer = messaging.Emailer('{}'.format(owner_email), testing=False)
        body = '''
 <head>
    <style>
    ul {{
        list-style: none;
        padding-left: 0;
    }}
    li {{
        margin-left: 0;
    }}
    </style>
 </head>
 <p>Hello!</p>

 <p>I've found some duplicate rows in: {}</p>

 <p>Would you be open to cleaning these up?</p>

 <p>I'd be happy to remove them for you via <a href="https://gist.github.com/d9fb06f2e1121447ea9ca918bf1c109e">this code</a>
 or help you incorporate this module into your own code.</p>

 <p>Here's a definition query of just the duplicates:</p>
 <p>OBJECTID IN ({})</p>

 <p>You could delete all features that match this query if you wanted to. It doesn't include the first instance of the duplicate group.</p>

 <p>Here's a full list of OBJECTIDs that have matching attributes including geometry (if applicable):</p>
 <ul>
    <li>{}</li>
 </ul>


 -Scott
        '''.format(basename(dataset), ', '.join([str(id) for id in dup_ids]), '</li><li>'.join([str(group) for group in duplicate_groups]))
        emailer.sendEmail('Duplicate features found in {}'.format(basename(dataset)), body)
        print(body)


 if __name__ == '__main__':
    import sys
    main(sys.argv[1], sys.argv[2])
	'''
	find_duplicates.py

	A module that contains code that finds duplicate features and notifies the owner of them.
	'''

	from os.path import basename

	import arcpy
	import messaging
	from xxhash import xxh64

	hashes = {}


	def is_skip_field(field):
	return 'SHAPE' in field.upper() or field.upper() in ['GLOBAL_ID', 'GLOBALID'] or field.startswith('OBJECTID')


	def get_fields(dataset):
	describe = arcpy.Describe(dataset)
	fields = [field.name for field in describe.fields if not is_skip_field(field.name)]

	if describe.datasetType.lower() != 'table':
	fields.append('SHAPE@WKT')
	fields.append('OID@')

	return fields


	def main(dataset, owner_email):
	print('looking for duplicates')
	with arcpy.da.SearchCursor(dataset, get_fields(dataset)) as cursor:
	for row in cursor:
	row_hash = xxh64(str(row[:-1])).hexdigest()
	group = hashes.setdefault(row_hash, [])
	group.append(row[-1])

	duplicate_groups = [group for group in hashes.values() if len(group) > 1]

	if len(duplicate_groups) > 0:
	dup_ids = []
	for group in duplicate_groups:
	dup_ids += group[1:]
	emailer = messaging.Emailer('{}'.format(owner_email), testing=False)
	body = '''
	<head>
	<style>
	ul {{
	list-style: none;
	padding-left: 0;
	}}
	li {{
	margin-left: 0;
	}}
	</style>
	</head>
	<p>Hello!</p>

	<p>I've found some duplicate rows in: {}</p>

	<p>Would you be open to cleaning these up?</p>

	<p>I'd be happy to remove them for you via <a href="https://gist.github.com/d9fb06f2e1121447ea9ca918bf1c109e">this code</a>
	or help you incorporate this module into your own code.</p>

	<p>Here's a definition query of just the duplicates:</p>
	<p>OBJECTID IN ({})</p>

	<p>You could delete all features that match this query if you wanted to. It doesn't include the first instance of the duplicate group.</p>

	<p>Here's a full list of OBJECTIDs that have matching attributes including geometry (if applicable):</p>
	<ul>
	<li>{}</li>
	</ul>


	-Scott
	'''.format(basename(dataset), ', '.join([str(id) for id in dup_ids]), '</li><li>'.join([str(group) for group in duplicate_groups]))
	emailer.sendEmail('Duplicate features found in {}'.format(basename(dataset)), body)
	print(body)


	if __name__ == '__main__':
	import sys
	main(sys.argv[1], sys.argv[2])
No results found