This script will finds sanji.log's duplicate line.

find_dunplicate

from sets import Set
import collections

lines = []
with open('/tmp/sanji.log', 'r') as f:
	lines = f.readlines()

boots = list()
bufferLines = list()
for line in lines:
	bufferLines.append(' '.join(line.split()[5:]))
	if line.find('Register Sanji controller OK. Model Name :::::::::: firmware ::::::::::') != -1:
		boots.append(bufferLines)
		bufferLines = list()

for boot in boots:
	print set([x for x in boot if boot.count(x) > 1]) # py 2.6
	print '\n'.join([x for x, y in collections.Counter(boot).items() if y > 1]) # py >= 2.7
	print '=' * 80

================================================================================
set(['', 'Deregister Sanji controller... end', 'Deregister Sanji controller Failed.', 'service.",', 'sanji_registration()', 'execute list: [', 'Register Sanji controller...', 'connect broker', 'model.",', 'start to connect broker', "{u'message': u'resource is locked'}", '423', 'end to connect broker, ret = 0', 'Start Model OK. Waiting for request.'])

Deregister Sanji controller... end
Deregister Sanji controller Failed.
423
sanji_registration()
service.",
model.",
Register Sanji controller...
execute list: [
connect broker
start to connect broker
{u'message': u'resource is locked'}
Start Model OK. Waiting for request.
end to connect broker, ret = 0
================================================================================
set(['', 'Deregister Sanji controller... end', 'service.",', 'sanji_registration()', 'execute list: [', 'Register Sanji controller...', 'connect broker', 'model.",', 'start to connect broker', 'end to connect broker, ret = 0', 'Start Model OK. Waiting for request.'])

Deregister Sanji controller... end
sanji_registration()
service.",
model.",
Register Sanji controller...
execute list: [
connect broker
start to connect broker
Start Model OK. Waiting for request.
end to connect broker, ret = 0
================================================================================