takes the plain internet census 2012 serviceprobes files on stdin (to be able to stream from the unpacker) and filters for status 1 + converts from quoted-printable to raw pcap files

convert_census_probes.py

#!/usr/bin/env python

# Author: Mark Schloesser ([email protected])
# Description: filter and convert internet census 2012 serviceprobes

# usage:
# convert_census_probes.py <port number> <output pcap path>
#  (uses the port number for the TCP header in the PCAP)

# example:
# unzpaq200 80-TCP_GetRequest-7.zpaq | python convert_census_probes.py 80 80-TCP_GetRequest-7-open.pcap

import sys
import quopri
import random
from scapy.all import IP,TCP,Raw,PcapWriter

def fakeip(inip):
	a,b,c,d = inip.split('.')
	return '10.{}.{}.{}'.format(b,c,d)

def main():
	try:
		portnum = int(sys.argv[1])
		pw = PcapWriter(sys.argv[2])
	except:
		print 'call this with <port number> <output pcap path>'
		return 1

	while True:
		l = sys.stdin.readline().strip()
		if not l: break

		# 4 columns: ip, timestamp, status code, data (if any)
		# filter all lines with status != 1

		columns = l.split()
		ip, timestamp, status = columns[:3]
		
		if status == '1':
			unquoted = ''
			if len(columns) > 3: unquoted = quopri.decodestring(columns[3])

			pkt = IP(src=ip, dst=fakeip(ip))/TCP(sport=portnum,dport=random.randint(1,65535))/Raw(unquoted)
			pw.write(pkt)
			
	pw.close()
	return 0

if __name__ == '__main__':
	try: sys.exit(main())
	except KeyboardInterrupt: pass

unzpaq200 is the ZPAQ reference decoder from its homepage: http://mattmahoney.net/dc/unzpaq200.cpp