Cassandra Load

gistfile1.py

import random
import json
import hashlib
import threading
import pycassa
import time
from pycassa.pool import ConnectionPool
from pycassa.columnfamily import ColumnFamily

reg = True
total = 100000000 #100 million
batch = 1

servers = [
  '', 
]

pool = ConnectionPool('ap', servers, pool_size=1)

def random_string(length):
  chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
  chars_len = len(chars)
  ret = ''
  for i in range(0, length):
    ret += chars[random.randint(0, chars_len - 1)]
  return ret

def generate_props():
  num_props = 100
  props = []

  for i in range(0, num_props):
    prop = {
      'ts': random.randint(0, 100000000) + 1280000000
    }
    
    rand = random.random()
    if rand < 0.1:
      prop['val'] = random.randint(0, 1000)
    elif rand < 0.45:
      prop['val'] = random.random()
    elif rand < 0.95:
      prop['val'] = (random.randint(0, 1) == 0)
    else:
      prop['val'] = random_string(100)
    
    props.append(json.dumps(prop))
  return props

def gen_key():
  global reg, total
  key = str(random.randint(0, total * total))

  if not reg:
    md5 = hashlib.md5();
    md5.update(key)
    key = md5.hexdigest()
  
  return key

def insert(reg, start, batch):
  rows = {}
  i = 0
  while i < batch:
    key = gen_key()
    
    row = {}
    p = 0
    for prop in generate_props():
      row['p' + str(p)] = prop
      p += 1
  
    rows[key] = row
    i += 1

  if reg:
    users = ColumnFamily(pool, 'RegisteredUsers')
  else:
    users = ColumnFamily(pool, 'AnonymousUsers')
  users.batch_insert(rows, write_consistency_level=pycassa.ConsistencyLevel.ANY)

def main(reg):
  global total, batch

  i = 0
  while i < total:
    st = time.time()
    insert(reg, i, batch)
    elapsed = time.time() - st
    qps = batch / elapsed
    i += batch
    print i, elapsed, qps

main(reg)