bowbowbow · December 1, 2018 08:34 · bowbowbow · Dec 1, 2018
diff --git a/doc_clustering_v2.py b/doc_clustering_v2.py
 import json, re, datetime
 import pandas as pd
 import nltk
 from nltk.stem.snowball import SnowballStemmer

 stopwords = nltk.corpus.stopwords.words('english')
 stemmer = SnowballStemmer("english")

 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.pipeline import FeatureUnion, Pipeline
 from sklearn.preprocessing import OneHotEncoder, LabelEncoder
 from sklearn.feature_extraction.text import TfidfVectorizer

 def data_load():
    frames = []
    for i in range(0, 8):
        with open('./data/koreaherald_1517_{}.json'.format(i), 'r') as f:
            data = json.load(f)
        df = pd.DataFrame.from_dict(data)
        # header:  [' author' ' body' ' description' ' section' ' time' 'title']
        # print(df.columns.values)
        frames.append(df)

    return pd.concat(frames)


 def chunking(text):
    chunks = []

    parser = nltk.RegexpParser("NP: {<DT>?<JJ>?<NN.*>*}")
    for sent in nltk.sent_tokenize(text):
        words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
        # words = [word for word in words if word not in stopwords]
        tags = nltk.pos_tag(words)
        tree = parser.parse(tags)
        leaves = [s.leaves() for s in tree.subtrees() if s.label() == "NP"]
        for leave in leaves:
            chunks.append([word[0] for word in leave])

    return chunks


 def tokenize_and_stem(text):
    stems = []
    for sent in nltk.sent_tokenize(text):
        words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
        words = [word for word in words if word not in stopwords]
        for word in words: stems.append(stemmer.stem(word))
    return stems


 def clustering(year):
    df = data_load()
    print(df.head())

    documents = []
    y = []
    for index, row in df.iterrows():
        time = datetime.datetime.strptime(row[' time'], '%Y-%m-%d %H:%M:%S')
        if year != time.year:
            continue
        title = row['title']
        documents.append(title)
        y.append(index)

    author_encoder = LabelEncoder()
    author_encoder.fit(df[' author'])
    df[' author'] = author_encoder.transform(df[' author'])

    class TextSelector(BaseEstimator, TransformerMixin):
        def __init__(self, key):
            self.key = key

        def fit(self, x, y=None):
            return self

        def transform(self, data_dict):
            return data_dict[self.key]

    class NumberSelector(BaseEstimator, TransformerMixin):
        def __init__(self, key):
            self.key = key

        def fit(self, x, y=None):
            return self

        def transform(self, data_dict):
            return data_dict[[self.key]]

    vectorizer = FeatureUnion(
        transformer_list=[
            ('title', Pipeline([
                ('selector', TextSelector(key='title')),
                ('tfidf', TfidfVectorizer(tokenizer=tokenize_and_stem, ngram_range=(1, 2)))
            ])),
            # ('author', Pipeline([
            #     ('selector', NumberSelector(key=' author')),
            #     ('onehot', OneHotEncoder())
            # ])),
        ],
        # weight components in FeatureUnion
        transformer_weights={
            'title': 0.7,
            # 'author': 0.3,
        },
    )

    X = vectorizer.fit_transform(df)
    print(vectorizer.transformer_list[0][1].named_steps['tfidf'].get_feature_names())

    true_k = 10
    from sklearn.cluster import KMeans
    model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1)
    model.fit(X)

    print("Top terms per cluster:")
    order_centroids = model.cluster_centers_.argsort()[:, ::-1]
    print('order_centroids : ', order_centroids)
    # terms = vectorizer.get_feature_names()
    terms = vectorizer.transformer_list[0][1].named_steps['tfidf'].get_feature_names()
    for i in range(true_k):
        keywords = [terms[ind] for ind in order_centroids[i, :10]]
        print('Cluster {}: {}'.format(i, ','.join(keywords)))


 if __name__ == "__main__":
    clustering(year=2017)
	import json, re, datetime
	import pandas as pd
	import nltk
	from nltk.stem.snowball import SnowballStemmer

	stopwords = nltk.corpus.stopwords.words('english')
	stemmer = SnowballStemmer("english")

	from sklearn.base import BaseEstimator, TransformerMixin
	from sklearn.pipeline import FeatureUnion, Pipeline
	from sklearn.preprocessing import OneHotEncoder, LabelEncoder
	from sklearn.feature_extraction.text import TfidfVectorizer

	def data_load():
	frames = []
	for i in range(0, 8):
	with open('./data/koreaherald_1517_{}.json'.format(i), 'r') as f:
	data = json.load(f)
	df = pd.DataFrame.from_dict(data)
	# header: [' author' ' body' ' description' ' section' ' time' 'title']
	# print(df.columns.values)
	frames.append(df)

	return pd.concat(frames)


	def chunking(text):
	chunks = []

	parser = nltk.RegexpParser("NP: {<DT>?<JJ>?<NN.>}")
	for sent in nltk.sent_tokenize(text):
	words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
	# words = [word for word in words if word not in stopwords]
	tags = nltk.pos_tag(words)
	tree = parser.parse(tags)
	leaves = [s.leaves() for s in tree.subtrees() if s.label() == "NP"]
	for leave in leaves:
	chunks.append([word[0] for word in leave])

	return chunks


	def tokenize_and_stem(text):
	stems = []
	for sent in nltk.sent_tokenize(text):
	words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
	words = [word for word in words if word not in stopwords]
	for word in words: stems.append(stemmer.stem(word))
	return stems


	def clustering(year):
	df = data_load()
	print(df.head())

	documents = []
	y = []
	for index, row in df.iterrows():
	time = datetime.datetime.strptime(row[' time'], '%Y-%m-%d %H:%M:%S')
	if year != time.year:
	continue
	title = row['title']
	documents.append(title)
	y.append(index)

	author_encoder = LabelEncoder()
	author_encoder.fit(df[' author'])
	df[' author'] = author_encoder.transform(df[' author'])

	class TextSelector(BaseEstimator, TransformerMixin):
	def __init__(self, key):
	self.key = key

	def fit(self, x, y=None):
	return self

	def transform(self, data_dict):
	return data_dict[self.key]

	class NumberSelector(BaseEstimator, TransformerMixin):
	def __init__(self, key):
	self.key = key

	def fit(self, x, y=None):
	return self

	def transform(self, data_dict):
	return data_dict[[self.key]]

	vectorizer = FeatureUnion(
	transformer_list=[
	('title', Pipeline([
	('selector', TextSelector(key='title')),
	('tfidf', TfidfVectorizer(tokenizer=tokenize_and_stem, ngram_range=(1, 2)))
	])),
	# ('author', Pipeline([
	# ('selector', NumberSelector(key=' author')),
	# ('onehot', OneHotEncoder())
	# ])),
	],
	# weight components in FeatureUnion
	transformer_weights={
	'title': 0.7,
	# 'author': 0.3,
	},
	)

	X = vectorizer.fit_transform(df)
	print(vectorizer.transformer_list[0][1].named_steps['tfidf'].get_feature_names())

	true_k = 10
	from sklearn.cluster import KMeans
	model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1)
	model.fit(X)

	print("Top terms per cluster:")
	order_centroids = model.cluster_centers_.argsort()[:, ::-1]
	print('order_centroids : ', order_centroids)
	# terms = vectorizer.get_feature_names()
	terms = vectorizer.transformer_list[0][1].named_steps['tfidf'].get_feature_names()
	for i in range(true_k):
	keywords = [terms[ind] for ind in order_centroids[i, :10]]
	print('Cluster {}: {}'.format(i, ','.join(keywords)))


	if __name__ == "__main__":
	clustering(year=2017)
No results found