findx.py

import dpkt
from socket import inet_ntoa
try:
    import cPickle as pickle
except:
    import pickle
from multiprocessing import Process, Queue

data = {'source':set([]),
            'destination':set([]),
            'len':set([]),
            'id':set([]),
            'off':set([]),
            'ttl':set([]),
            'p':set([]),
            'sum':set([]),
            'payload':set([]),
            'options':set([]),
            'type':set([])}

def getData(frame):
    global data
    ip = frame.data
    if "10.0.0.3" != inet_ntoa(ip.src) and "10.0.0.3" != inet_ntoa(ip.dst):
        pass
    data['source'].add(inet_ntoa(ip.src))
    data['destination'].add(inet_ntoa(ip.dst))
    data['len'].add(ip.len)
    data['id'].add(ip.id)
    data['off'].add(ip.off)
    data['ttl'].add(ip.ttl)
    data['p'].add(ip.p)
    data['sum'].add(ip.sum)
    data['payload'].add(repr(ip.data))
    data['options'].add(ip.opts)
    data['type'].add(frame.type)
    
    
if __name__ == '__main__':
                
    bytesProcessed = 0
    toProcess = 36489425360
    
    with open("alldata2.pcap", 'rb') as wiresharkData:
    
        pcap = dpkt.pcap.Reader(wiresharkData)

        for timestamp, buf in pcap:
            bytesProcessed += len(buf)
            print("\r%.2f percent processed"%(bytesProcessed/float(toProcess)*100), end='\r')
            try:
                eth = dpkt.ethernet.Ethernet(buf)
                
                if not isinstance(eth.data, dpkt.ip.IP):
                    pass
                data.append(eth)
            except:
                pass
    
    p = multiprocessing.Pool(multiprocessing.cpu_count() - 2)
    p.map(getData, data)    
    
    with open("findx.pickle", "a+") as infoFile:
        pickle.dump(info, infoFile)