laserson

# The Python avro client expects a seekable Avro data file, which makes it annoying
# to stream bytes through it using HDFS clients that just give you cat (like snakebite).
# It's idiotic because the client only seeks to the end in order to call tell() to get
# the file size, which in turn is only used to determine when you get to EOF.

import snakebite.client

class AvroStreamWrapper(object):
    # this class can be provided to DataFileReader to read Avro data.
    def __init__(self, hdfs_client, path):

mallamanis

#!/usr/bin/env python
import nltk
from nltk.stem.porter import PorterStemmer


def preprocessText(text):
	# To lower case
	text = text.lower();

	# Tokenize text

rgaidot

import nltk

text =  """Barack Hussein Obama II  (born August 4, 1961) is the 44th and current President of the United States. He is the first African American to hold the office. Obama previously served as a United States Senator from Illinois, from January 2005 until he resigned after his election to the presidency in November 2008."""

sentences = nltk.sent_tokenize(text)
tokenized_sentences = [nltk.word_tokenize(sentence) for sentence in sentences]
tagged_sentences = [nltk.pos_tag(sentence) for sentence in tokenized_sentences]
chunked_sentences = nltk.batch_ne_chunk(tagged_sentences, binary=True)

def extract_entity_names(t):