Test GCE disk throughput

gce-disk-throughput.py

#!/usr/bin/python

''' Used to troubleshoot Google Compute Engine slow disk throughput '''

''' Adjust block device readahead to optimize throughput via '''
''' $ blockdev [/dev/sda] --setra $((1024*1024*8)) '''

import sys
import os
import datetime
import time

def usage():
 print "Usage: " + str(sys.argv[0]) + " <source file> <dest file> [buffer size]"
 sys.exit()

if len(sys.argv) < 3:
 usage()

_srcfile = sys.argv[1]
_dstfile = sys.argv[2]

try:
 _buffsize = int(sys.argv[3])
except:
 _buffsize = 1024*1024*8
 pass

if not os.path.isfile(_srcfile):
 usage()

if os.path.isfile(_dstfile):
 os.remove(_dstfile)

_starttime = time.time()
print "Current timestamp: " + str(_starttime)

fsize = os.path.getsize(_srcfile)
_dfh = open(_dstfile,'wb')

with open(_srcfile,'rb') as _sfh:

 while _sfh.tell() < fsize:

  _precopy = datetime.datetime.now()
  buff = _sfh.read(_buffsize)
  _postread = datetime.datetime.now()

  if not buff:
   break

  _dfh.write(buff)
  os.fsync(_dfh)
  _postcopy = datetime.datetime.now()
  _postcopytime = time.time()
  _pstamp = (_postcopytime - _starttime)
  _sizecopied = os.path.getsize(_dstfile) / (1024*1024)

  if _pstamp > 1:
   _MBps = float(_sizecopied / _pstamp)

  else:
   _MBps = float(0)
  _cstamp = float(float((_postcopy - _precopy).microseconds) / float(1000000))
  _curMBps = float((_buffsize / (1024*1024)) / _cstamp)
  _prtime = float(float((_postread - _precopy).microseconds) / float(1000))
  _pwtime = float(float((_postcopy - _postread).microseconds) / float(1000))


  print "Read " + str(_buffsize) + " bytes in " + str(_prtime) + " ms."
  print "Wrote " + str(_buffsize) + " bytes in " + str(_pwtime) + " ms."
  print "Wrote " + str(_sizecopied) + " MB in " + str(_pstamp) + " seconds. [AVG: " + str(_MBps) + " MB/s, CUR: " + str(_curMBps) + " MB/s]"