Pinak-Chakraborty

import sys, os, os.path, glob, codecs

# Set the codecs
sys.stdout = codecs.getwriter('UTF-8')(sys.stdout.detach())

# set the delimiters
delimiterSet = ";.,!?\"()':[]\n/+-—=≤≥{}><*’”“|"
digits = "0123456789"
chars = "abcdefghijklmnopqrstuvwxyz"
chars = "".join( (chars, chars.upper()) )

Pinak-Chakraborty

import sys, os, os.path, glob, codecs

# Set the codecs
sys.stdout = codecs.getwriter('UTF-8')(sys.stdout.detach())

# no of highest frequemcy unigrams and bigrams that will be written out
writemax = 100

def wordTokenizier(line):
    #delimiters = "[A-Z]{2,}(?![a-z])|[A-Z][a-z]+(?=[A-Z])|[\'\w\-]+"

Pinak-Chakraborty

import re, random, numpy as np

def centers (X, K):
    # Initialize to K random centers
    oldmu = random.sample(list(X), K)
    mu = random.sample(list(X), K)
    while not has_converged(mu, oldmu):
        oldmu = mu
        # Assign all points in X to clusters
        clusters = cluster_points(X, mu)

Pinak-Chakraborty

#-------------------------------------------------------------------------------
#
# This module determines the sentence probability for all sentences in the test
# data set. It uses deleted inerpolation as: (for bigram)
#
#   prop(w1,w2,w3) = coff1*(trigram freq(w1,w2,w3)/bigram freg(w1,w2))+ (
#               (coff2*(bigram freq(w1,w2)/unigram freq(w1) +
#                (coff3*(unigram freq(w1)/total no of unigram))
#
#-------------------------------------------------------------------------------

Pinak-Chakraborty

#-------------------------------------------------------------------------------
#
# This module determines the sentence probability for all sentences in the test
# data set. It uses Back off method in determing the probability
#
#   prob(w1,w2,w3) is calculated as follows:
#
#   if trigram count(w1,w2,w3) not zero
#       prob (w1,w2,w3) = coff1*trigram count (w1,w2,w3)/bigram count (w1,w2)
#   else if bigram count (w1,w2) not zero

Pinak-Chakraborty

Pinak-Chakraborty

Pinak-Chakraborty

node
financial_service
"1874"
"16,000_atm"
"4,700_financial_center"
account
asset-based_lending
bank_of_america_corporation
banking
brokerage

Pinak-Chakraborty