tsne-100m.py

from bhtsne import tsne
from sklearn.datasets.base import load_data
from sklearn.utils import Bunch
import pandas as pd
import numpy as np
from ggplot import *


def load_netflix(return_X_y=False):
    data, target, target_names = load_data('/home/comtom/extra/netflix/', 'file1-100mil.csv')

    if return_X_y:
        return data, target

    return Bunch(data=data, target=target,
                 target_names=[1, 2, 3, 4, 5],
                 DESCR='',
                 feature_names=['movieId', 'userID', 'ratingDate'])

netflix = load_netflix()

df = pd.DataFrame(netflix.target, columns=['rating'])
rndperm = np.random.permutation(df.shape[0])

Y = tsne(netflix.data)

df_tsne = df.loc[rndperm[:99999], :].copy()
df_tsne['x-tsne'] = Y[:, 0]
df_tsne['y-tsne'] = Y[:, 1]

chart = ggplot(df_tsne, aes(x='x-tsne', y='y-tsne', color='rating')) \
        + geom_point(size=30, alpha=0.5) \
        + ggtitle("tSNE dimensions colored by rating")

chart