Read web page

read-from-redis.py

import redis
import requests
from bs4 import BeautifulSoup

#create redis connection
redis_server = redis.Redis("localhost")

#read a link from the redis index
requrl = redis_server.lindex("index",2)

#parse up the content
r = requests.get(requrl)
soup = BeautifulSoup(r.text)

#holds all the page qualifying data
qualifier = {}

#for all the content in the page
for content in soup.find_all('p'):
	words = content.get_text().split()
	#add to dictionary. If it doesnt exist add it with an instance of 1 else increment it
	for word in words:
		if not word in qualifier:
			qualifier[word] = 1
		else:
			qualifier[word] += 1