str-tests.py

#!/usr/bin/env python
from __future__ import unicode_literals

import gc
import os
import sys
import psutil
import time
from io import BytesIO


process = psutil.Process(os.getpid())

last_time = time.clock()
print 'RSS: %s bytes' % process.memory_info().rss
print
print 'Reading lines from data file'

with open('datafile', 'r') as fp:
    src_lines = fp.readlines()


print '%s lines' % len(src_lines)


now = time.clock()
print 'Elapsed: %s seconds' % (now - last_time)
last_time = now

print 'RSS: %s bytes' % process.memory_info().rss
print


if sys.argv[1] == 'concat':
    print 'Concatenating strings'
    data = b''

    for line in src_lines:
        data += line + b'\n'
        print 'RSS: %s bytes' % process.memory_info().rss
elif sys.argv[1] == 'listjoin':
    print 'Building and joining list'
    data = []

    for line in src_lines:
        data.append(line)
        data.append('\n')
        print 'RSS: %s bytes' % process.memory_info().rss

    now = time.clock()
    print 'Elapsed: %s seconds' % (now - last_time)
    last_time = now
    print 'RSS (new list, pre-join): %s bytes' % process.memory_info().rss

    data = b''.join(data)
elif sys.argv[1] == 'bytesio':
    print 'Building BytesIO'
    data = BytesIO()

    for line in src_lines:
        data.write(line)
        data.write(b'\n')
        print 'RSS: %s bytes' % process.memory_info().rss

    data = data.getvalue()


now = time.clock()
print 'Elapsed: %s seconds' % (now - last_time)

gc.collect()
print 'RSS: %s bytes' % process.memory_info().rss