redis python hll

gistfile1.txt

# coding: utf-8

# In[ ]:

# https://gist.github.com/DavidJFelix/113fad6a0a7affdd880d
import redis


# In[2]:

import nltk


# In[4]:

pool = redis.ConnectionPool(host="127.0.0.1", port=6379, db=0)
r = redis.Redis(connection_pool=pool)


# In[5]:

r.set('breakfast', 'spam')
r.get('breakfast')


# In[ ]:

import nltk
nltk.download()


# In[ ]:

caesar = nltk.corpus.gutenberg.words('shakespeare-caesar.txt')


# In[ ]:

print "Words in Julias Caesar:", len(caesar)
hamlet = nltk.corpus.gutenberg.words('shakespeare-hamlet.txt')
print "Words in Hamlet:", len(hamlet)
macbeth = nltk.corpus.gutenberg.words('shakespeare-macbeth.txt')
print "Words in MacBeth:", len(macbeth)


# In[ ]:

r.pfadd('test', 'spam', 'spam', 'spam', 'spam', 'eggs')
r.pfcount('test')


# In[ ]:

r.delete('caesar')
r.delete('hamlet')
r.delete('macbeth')

from time import time
t0 = time()

r.pfadd('caesar', *caesar)
r.pfadd('hamlet', *hamlet)
r.pfadd('macbeth', *macbeth)
print "Unique words in Julias Caesar:", r.pfcount('caesar')
print "Unique words in Hamlet:", r.pfcount('hamlet')
print "Uniqe words in MacBeth:", r.pfcount('macbeth')

t1 = time()

print "It took", t1-t0, "seconds to count this."
print "Julias Caesar is using", len(r.dump('caesar')), "bytes in memory, but is", len(r.get('caesar')), "bytes"
print "Hamlet is using", len(r.dump('hamlet')), "bytes in memory, but is", len(r.get('hamlet')), "bytes"
print "MacBeth is using", len(r.dump('macbeth')), "bytes in memory, but is", len(r.get('macbeth')), "bytes"


# In[ ]:

r.delete('shakespeare')

t0 = time()

r.pfmerge('shakespeare', 'caesar', 'hamlet', 'macbeth')
print "Unique words in Julias Caesar, Hamlet and Macbeth:", r.pfcount('shakespeare')

t1 = time()

print "It took", t1-t0, "seconds to merge all 3 HyperLogLogs into 1"


# In[ ]:

import sys

t0 = time()

unique_caesar = set(caesar)
unique_hamlet = set(hamlet)
unique_macbeth = set(macbeth)

print "Unique words in Julia Caesar, counted:", len(unique_caesar)
print "Unique words in Hamlet, counted:", len(unique_hamlet)
print "Unique words in Macbeth, counted:", len(unique_macbeth)

t1 = time()

print "It took python", t1-t0, "seconds to count unique elements."
print "Julias Caesar is using", sys.getsizeof(unique_caesar), "bytes"
print "Hamlet is using", sys.getsizeof(unique_hamlet), "bytes"
print "MacBeth is using", sys.getsizeof(unique_macbeth), "bytes"


# In[ ]:

r.delete('caesar')
r.delete('hamlet')
r.delete('macbeth')

from time import time
import sys

t0 = time()

r.sadd('caesar', *caesar)
r.sadd('hamlet', *hamlet)
r.sadd('macbeth', *macbeth)
print "Unique words in Julias Caesar:", r.scard('caesar')
print "Unique words in Hamlet:", r.scard('hamlet')
print "Uniqe words in MacBeth:", r.scard('macbeth')

t1 = time()

print "It took", t1-t0, "seconds to count this."
print "Julias Caesar is using", len(r.dump('caesar')), "bytes"
print "Hamlet is using", len(r.dump('hamlet')), "bytes"
print "MacBeth is using", len(r.dump('macbeth')), "bytes"


# In[ ]:

from numpy import random
r.delete('test')

for x in xrange(1000000):
    for dupe in xrange(random.randint(1, 10)):
        r.pfadd('test', str(x))

r.pfcount('test')


# In[ ]:

len(r.get('test'))


# In[ ]:

items = [
        'apple',
        'banana',
        'cucumber',
        'dogfood',
        'eggplant',
        'flowers',
        'goat cheese',
        'hummus',
        'iceberg lettuce',
        'jalepenos'
    ]


def shopping_list_controller(req_url, req_ip):
   r.pfadd('ip-' + req_url, req_ip)
   item_list = list()

    # Shopping lists are only 4-9 items here
    for x in xrange(random.randint(4, 10)):
        item_list.append(random.choice(items))

    # Ghetto hash function
    item_list.sort()

    r.pfadd('list-' + req_url, str(item_list))
    r.pfadd('ip-list-' + req_url, str(item_list) + req_ip)

    
# Don't tell anyone... but all of my fake services do the same thing
recommended_items_controller = shopping_list_controller
cheaper_alternatives_controller = shopping_list_controller
recipe_items_controller = shopping_list_controller

routes = {
    '/list/': shopping_list_controller,
    '/recomended/': recommended_items_controller,
    '/cheapalts/': cheaper_alternatives_controller,
    '/recipe/': recipe_items_controller
}


# In[ ]:

for url in routes.keys():
    r.delete('ip-' + url)
    r.delete('list-' + url)
    r.delete('ip-list-' + url)

def rand_ip():
    return str(random.randint(0,256)) + "." + str(random.randint(0,256))

for x in xrange(100000):
    URL = random.choice(routes.keys())
    routes[URL](URL, rand_ip())


# In[ ]:

for url in routes.keys():
    print "Unique IPs on", url, ":", r.pfcount('ip-' + url)
    print "Unique lists on", url, ":", r.pfcount('list-' + url)
    print "Unique IP/list on", url, ":", r.pfcount('ip-list-' + url)


# On the fly, merge IP recordings into one new sitewide metric
r.delete("ip")
r.pfmerge("ip", *["ip-" + key for key in routes.keys()])
print "Unique IPs, sitewide:", r.pfcount("ip")

# On the fly, merge list recordings into one new sitewide metric
r.delete("list")
r.pfmerge("list", *["list-" + key for key in routes.keys()])
print "Unique lists, sitewide", r.pfcount("list")

# On the fly, merge IP/list recordings into onew new sitewide metric
r.delete("ip-list")
r.pfmerge("ip-list", *["ip-list-" + key for key in routes.keys()])
print "Unique IP/lists, sitewide", r.pfcount("ip-list")


# In[ ]: