renren_split_for_sorting.py

# -*- coding: utf-8 -*-
"""
Spyder Editor

This temporary script file is located here:
D:\chengjun\WinPython-64bit-2.7.6.4\settings\.spyder2\.temp.py
"""

'''
# Step2: split the duplicated data into about 2000+ files by user ids
# to prepare for deleting the duplicated ties
''' 
path = "D:/renren/"

f = open(path + "friends_all.txt")

n = 0
for line in f:
    From, To, time = line.strip().split('\t')
    n += 1
    if n%10000 == 0:
        print n, From, To, time
    From = int(From)
    To = int(To)
    From, To = sorted([From, To])
    record = str(From) + '\t' + str(To) + '\t' + time
    file_save = path + "friends_sorted/" + str(From/10000)
    with open(file_save,'a') as p:
        p.write(record+"\n")
 
 
'''
# Step1 detect wheter the data is duplicated
'''   
# the 14 G data is duplicated:
# 1, 2
# 2, 1
    
#path = "D:/renren/"
#f = open(path + "friends_all.txt")    
#n = 0
#for line in f:
#    From, To, time = line.strip().split('\t')
#    n += 1
#    if n%6000 == 0:
#        print n, From, To, time
#    if From == "3" and To == "1":
#        print "double"
#        break