remove_duplicates.py

remove_duplicates.py

'''
remove_duplicates.py

A module that removes duplicate features from a feature class.

Usage:
from remove_duplicates import remove

remove(r'path/to/data')

or via command line:
python remove_duplicates.py "path/to/data"
'''
import arcpy
from xxhash import xxh64

hashes = []
removed = []
cc_email = 'stdavis@utah.gov'


def is_skip_field(field):
    return 'SHAPE' in field.upper() or field.upper() in ['GLOBAL_ID', 'GLOBALID'] or field.startswith('OBJECTID')


def get_fields(dataset):
    fields = [field.name for field in arcpy.Describe(dataset).fields if not is_skip_field(field.name)]
    fields.append('SHAPE@WKT')
    fields.append('OID@')

    return fields


def remove(dataset):
    print('looking for duplicates')
    total_count = int(arcpy.management.GetCount(dataset)[0])
    with arcpy.da.UpdateCursor(dataset, get_fields(dataset)) as cursor:
        count = 0
        step = 5
        current_step = step
        for row in cursor:
            count += 1
            row_hash = xxh64(str(row[:-1])).hexdigest()
            if row_hash not in hashes:
                hashes.append(row_hash)
            else:
                removed.append(row[-1])
                cursor.deleteRow()

            if count/float(total_count)*100 > current_step:
                print('{}%'.format(current_step))
                current_step += step

    if len(removed) > 0:
        print('removed features with the following OBJECTIDs: {}'.format(', '.join([str(id) for id in removed])))
    else:
        print('no duplicates found')


if __name__ == '__main__':
    import sys
    remove(sys.argv[1])