bowbowbow · December 2, 2018 15:29 · bowbowbow · Dec 2, 2018
diff --git a/doc_clustering_k=20.py b/doc_clustering_k=20.py
 import json, re, datetime
 import pandas as pd
 import numpy as np
 import nltk
 from nltk.stem.snowball import SnowballStemmer

 stopwords = nltk.corpus.stopwords.words('english')
 stemmer = SnowballStemmer("english")

 from sklearn.base import BaseEstimator, TransformerMixin
 from sklearn.pipeline import FeatureUnion, Pipeline
 from sklearn.preprocessing import OneHotEncoder, LabelEncoder
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics import pairwise_distances_argmin_min, pairwise


 def data_load():
    frames = []
    for i in range(0, 8):
        with open('./data/koreaherald_1517_{}.json'.format(i), 'r') as f:
            data = json.load(f)
        data['year'] = dict()
        data['timestamp'] = dict()
        for doc_id in data[' body']:
            time = datetime.datetime.strptime(data[' time'][doc_id], '%Y-%m-%d %H:%M:%S')
            data['year'][doc_id] = time.year
            data['timestamp'][doc_id] = (time.month * 30 + time.day) / 100.0
            data['title'][doc_id] = data['title'][doc_id].replace(r"\[.*\]","")

        df = pd.DataFrame.from_dict(data)
        # header:  [' author' ' body' ' description' ' section' ' time' 'title']
        # print(df.columns.values)
        frames.append(df)

    return pd.concat(frames)


 def chunking(text):
    chunks = []

    parser = nltk.RegexpParser("NP: {<DT>?<JJ>?<NN.*>*}")
    for sent in nltk.sent_tokenize(text):
        words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
        # words = [word for word in words if word not in stopwords]
        tags = nltk.pos_tag(words)
        tree = parser.parse(tags)
        leaves = [s.leaves() for s in tree.subtrees() if s.label() == "NP"]
        for leave in leaves:
            chunk = [word[0] for word in leave]
            chunks.append('_'.join(chunk))

    return chunks


 def get_proppers(text):
    propernouns = []
    for sent in nltk.sent_tokenize(text):
        words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
        tags = nltk.pos_tag(words)
        propernouns += [word for word, pos in tags if pos == 'NNP' or pos == 'NNPS']
    return propernouns


 def tokenize_and_stem(text):
    stems = []
    for sent in nltk.sent_tokenize(text):
        words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
        words = [word for word in words if word not in stopwords]
        for word in words: stems.append(stemmer.stem(word))
    return stems


 def clustering(year):
    print('year :', year)
    df = data_load()
    df = df[df.year == year]
    # print(df[:5].to_string())

    encoder = LabelEncoder()
    encoder.fit(df[' author'])
    df[' author'] = encoder.transform(df[' author'])
    encoder.fit(df[' section'])
    df[' section'] = encoder.transform(df[' section'])

    class TextSelector(BaseEstimator, TransformerMixin):
        def __init__(self, key):
            self.key = key

        def fit(self, x, y=None):
            return self

        def transform(self, data_dict):
            return data_dict[self.key]

    class NumberSelector(BaseEstimator, TransformerMixin):
        def __init__(self, key):
            self.key = key

        def fit(self, x, y=None):
            return self

        def transform(self, data_dict):
            return data_dict[[self.key]]



    vectorizer = FeatureUnion(
        transformer_list=[
            ('propernouns', Pipeline([
                ('selector', TextSelector(key=' body')),
                ('tfidf', TfidfVectorizer(tokenizer=get_proppers, ngram_range=(1, 2)))
            ])),
            ('title', Pipeline([
                ('selector', TextSelector(key='title')),
                ('tfidf', TfidfVectorizer(max_df=300, tokenizer=tokenize_and_stem, ngram_range=(1, 2)))
            ])),
            ('body', Pipeline([
                ('selector', TextSelector(key=' body')),
                ('tfidf', TfidfVectorizer(max_df=350, tokenizer=tokenize_and_stem, ngram_range=(1, 2)))
            ])),
            ('author', Pipeline([
                ('selector', NumberSelector(key=' author')),
                ('onehot', OneHotEncoder(categories='auto'))
            ])),
            ('section', Pipeline([
                ('selector', NumberSelector(key=' section')),
                ('onehot', OneHotEncoder(categories='auto'))
            ])),
            ('timestamp', Pipeline([
                ('selector', NumberSelector(key='timestamp')),
            ])),
        ],
        # weight components in FeatureUnion
        transformer_weights={
            'section': 1.2,
            'propernouns': 2.0,
            'title': 1.0,
            'body': 1.0,
            'timestamp': 0.5,
            'author': 0.3,
        },
    )

    X = vectorizer.fit_transform(df)

    true_k = 20
    from sklearn.cluster import KMeans
    model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1)
    model.fit(X)

    df['cluster'] = model.labels_

    import collections
    counts = collections.Counter(model.labels_)
    counts = [(c, counts[c]) for c in counts]
    counts = sorted(counts, key=lambda x: x[1], reverse=True)
    closest, _ = pairwise_distances_argmin_min(model.cluster_centers_, X)
    print('closest :', closest)

    for count in counts[:10]:
        c = count[0]
        print('cluster {}'.format(c))

        dis = model.transform(X)[:, c]
        dis = [(i, dis[i]) for i in range(len(dis))]
        dis = sorted(dis, key=lambda x: x[1])

        for item in dis[:5]:
            doc_id = item[0]
            print(doc_id, ', title :', df.iloc[doc_id]['title'])

        # print(chunking(doc[' body']))


 if __name__ == "__main__":
    clustering(year=2017)
	import json, re, datetime
	import pandas as pd
	import numpy as np
	import nltk
	from nltk.stem.snowball import SnowballStemmer

	stopwords = nltk.corpus.stopwords.words('english')
	stemmer = SnowballStemmer("english")

	from sklearn.base import BaseEstimator, TransformerMixin
	from sklearn.pipeline import FeatureUnion, Pipeline
	from sklearn.preprocessing import OneHotEncoder, LabelEncoder
	from sklearn.feature_extraction.text import TfidfVectorizer
	from sklearn.metrics import pairwise_distances_argmin_min, pairwise


	def data_load():
	frames = []
	for i in range(0, 8):
	with open('./data/koreaherald_1517_{}.json'.format(i), 'r') as f:
	data = json.load(f)
	data['year'] = dict()
	data['timestamp'] = dict()
	for doc_id in data[' body']:
	time = datetime.datetime.strptime(data[' time'][doc_id], '%Y-%m-%d %H:%M:%S')
	data['year'][doc_id] = time.year
	data['timestamp'][doc_id] = (time.month * 30 + time.day) / 100.0
	data['title'][doc_id] = data['title'][doc_id].replace(r"\[.*\]","")

	df = pd.DataFrame.from_dict(data)
	# header: [' author' ' body' ' description' ' section' ' time' 'title']
	# print(df.columns.values)
	frames.append(df)

	return pd.concat(frames)


	def chunking(text):
	chunks = []

	parser = nltk.RegexpParser("NP: {<DT>?<JJ>?<NN.>}")
	for sent in nltk.sent_tokenize(text):
	words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
	# words = [word for word in words if word not in stopwords]
	tags = nltk.pos_tag(words)
	tree = parser.parse(tags)
	leaves = [s.leaves() for s in tree.subtrees() if s.label() == "NP"]
	for leave in leaves:
	chunk = [word[0] for word in leave]
	chunks.append('_'.join(chunk))

	return chunks


	def get_proppers(text):
	propernouns = []
	for sent in nltk.sent_tokenize(text):
	words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
	tags = nltk.pos_tag(words)
	propernouns += [word for word, pos in tags if pos == 'NNP' or pos == 'NNPS']
	return propernouns


	def tokenize_and_stem(text):
	stems = []
	for sent in nltk.sent_tokenize(text):
	words = [word for word in nltk.word_tokenize(sent) if word.isalpha()]
	words = [word for word in words if word not in stopwords]
	for word in words: stems.append(stemmer.stem(word))
	return stems


	def clustering(year):
	print('year :', year)
	df = data_load()
	df = df[df.year == year]
	# print(df[:5].to_string())

	encoder = LabelEncoder()
	encoder.fit(df[' author'])
	df[' author'] = encoder.transform(df[' author'])
	encoder.fit(df[' section'])
	df[' section'] = encoder.transform(df[' section'])

	class TextSelector(BaseEstimator, TransformerMixin):
	def __init__(self, key):
	self.key = key

	def fit(self, x, y=None):
	return self

	def transform(self, data_dict):
	return data_dict[self.key]

	class NumberSelector(BaseEstimator, TransformerMixin):
	def __init__(self, key):
	self.key = key

	def fit(self, x, y=None):
	return self

	def transform(self, data_dict):
	return data_dict[[self.key]]



	vectorizer = FeatureUnion(
	transformer_list=[
	('propernouns', Pipeline([
	('selector', TextSelector(key=' body')),
	('tfidf', TfidfVectorizer(tokenizer=get_proppers, ngram_range=(1, 2)))
	])),
	('title', Pipeline([
	('selector', TextSelector(key='title')),
	('tfidf', TfidfVectorizer(max_df=300, tokenizer=tokenize_and_stem, ngram_range=(1, 2)))
	])),
	('body', Pipeline([
	('selector', TextSelector(key=' body')),
	('tfidf', TfidfVectorizer(max_df=350, tokenizer=tokenize_and_stem, ngram_range=(1, 2)))
	])),
	('author', Pipeline([
	('selector', NumberSelector(key=' author')),
	('onehot', OneHotEncoder(categories='auto'))
	])),
	('section', Pipeline([
	('selector', NumberSelector(key=' section')),
	('onehot', OneHotEncoder(categories='auto'))
	])),
	('timestamp', Pipeline([
	('selector', NumberSelector(key='timestamp')),
	])),
	],
	# weight components in FeatureUnion
	transformer_weights={
	'section': 1.2,
	'propernouns': 2.0,
	'title': 1.0,
	'body': 1.0,
	'timestamp': 0.5,
	'author': 0.3,
	},
	)

	X = vectorizer.fit_transform(df)

	true_k = 20
	from sklearn.cluster import KMeans
	model = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1)
	model.fit(X)

	df['cluster'] = model.labels_

	import collections
	counts = collections.Counter(model.labels_)
	counts = [(c, counts[c]) for c in counts]
	counts = sorted(counts, key=lambda x: x[1], reverse=True)
	closest, _ = pairwise_distances_argmin_min(model.cluster_centers_, X)
	print('closest :', closest)

	for count in counts[:10]:
	c = count[0]
	print('cluster {}'.format(c))

	dis = model.transform(X)[:, c]
	dis = [(i, dis[i]) for i in range(len(dis))]
	dis = sorted(dis, key=lambda x: x[1])

	for item in dis[:5]:
	doc_id = item[0]
	print(doc_id, ', title :', df.iloc[doc_id]['title'])

	# print(chunking(doc[' body']))


	if __name__ == "__main__":
	clustering(year=2017)
No results found