Remove duplicate json files

jsonduperemover.py

# You can remove duplicate json files (for example if you created them by accident or if avoiding them is to complex)

import json
from os import listdir, remove
from copy import copy


file_swith = 'ml-winners' # Empty string selects all files, or select only files by matching their starting
endswith = '.json' # or alternatively change the ending and leave file_swith empty
old_content = None

file_list = listdir()
file_list.sort()
for file in file_list:
	if file.endswith(endswith):
		if file.startswith(file_swith):
			with open(file, encoding = 'utf-8') as f:
				data = json.loads(f.read())
			if data != old_content:
				print(f'File {file!r} is different from the previous one. (FALSE)')
			else:
				print(f'File {file!r} is the same as the previous one. (TRUE)')
				remove(file)
				print('File removed')
			old_content = copy(data)