chyikwei

import bz2
import os
import re
import random
import csv
from urlparse import urlparse

#old bz2 files
#path = '/logs/prod/nginx/2013/06'
path = '/var/log/remote/prod/nginx/2013/08'

chyikwei

import random as rand
from string import Template
from itertools import repeat
from datetime import datetime, timedelta

from apps.glue.util.date import start_of_next_month
from apps.dashboard.views import get_bigquery, get_signup_tables, get_interaction_tables
from apps.dashboard.views import datetime_human, date_range2, date_name

bq = get_bigquery()

chyikwei

import pandas as pd 
import numpy as np

from sklearn import metrics
from sklearn import cross_validation

# models
from sklearn import linear_model
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier

chyikwei

import pandas as pd 
import numpy as np

from sklearn import metrics
from sklearn import cross_validation

# models
from sklearn import linear_model
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import RandomForestClassifier

chyikwei

File: lda.py
Function: _em_step at line 233
Total time: 144.169 s

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
   233                                               @profile
   234                                               def _em_step(self, X, batch_update):
   235                                                   """
   236                                                   EM update for 1 iteration

chyikwei

File: lda.py
Function: _dirichlet_expectation at line 24
Total time: 8.96912 s

Line #      Hits         Time  Per Hit   % Time  Line Contents
==============================================================
    24                                           @profile
    25                                           def _dirichlet_expectation(alpha):
    26                                               """
    27                                               For a vector theta ~ Dir(alpha), computes E[log(theta)] given alpha.

chyikwei

from time import time
import logging
import numpy as np
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

from gensim.matutils import Sparse2Corpus
#from gensim.models.ldamodel import LdaModel
from gensim.models.ldamulticore import LdaMulticore

chyikwei

chyikwei@:~/github/scikit-learn (onlineldavb)$ pep8 sklearn/decomposition/online_lda.py
sklearn/decomposition/online_lda.py:54:80: E501 line too long (81 > 79 characters)
sklearn/decomposition/online_lda.py:73:80: E501 line too long (84 > 79 characters)
sklearn/decomposition/online_lda.py:149:80: E501 line too long (80 > 79 characters)
sklearn/decomposition/online_lda.py:157:80: E501 line too long (84 > 79 characters)
sklearn/decomposition/online_lda.py:168:80: E501 line too long (89 > 79 characters)
sklearn/decomposition/online_lda.py:172:80: E501 line too long (93 > 79 characters)
sklearn/decomposition/online_lda.py:174:80: E501 line too long (90 > 79 characters)
sklearn/decomposition/online_lda.py:175:80: E501 line too long (94 > 79 characters)
sklearn/decomposition/online_lda.py:176:80: E501 line too long (95 > 79 characters)

chyikwei

chyikwei

sudo apt-get update
sudo apt install htop
wget http://us.download.nvidia.com/tesla/375.51/nvidia-driver-local-repo-ubuntu1604_375.51-1_amd64.deb
sudo dpkg -i nvidia-driver-local-repo-ubuntu1604_375.51-1_amd64.deb
sudo apt-get -y install cuda-drivers
sudo reboot

sudo apt-get install python-pip python-dev build-essential
sudo pip install virtualenv
sudo pip install --upgrade pip setuptools