VibhuJawa

VibhuJawa

#!/bin/bash

set -eu

wget -nc https://github.com/rapidsai/notebooks-extended/raw/master/utils/env-check.py
echo "Checking for GPU type:"
python env-check.py

if [ ! -f Miniconda3-4.5.4-Linux-x86_64.sh ]; then
    echo "Removing conflicting packages, will replace with RAPIDS compatible versions"

VibhuJawa

normalized_count_array = count_dary/np.sum(count_dary,axis=1)[:,None]

VibhuJawa

VibhuJawa

## chunksize = 1.9 M, 10% of dataset
small_df = df.head(1_900_000).copy()
%time ouptput_df = preprocess_text_df(small_df,filter_regex=filters_regex)

VibhuJawa

## chunksize = 950K, 5% of dataset
small_df = df.head(950_000).copy(deep=True)
%time ouptput_df = preprocess_text_df(small_df,filter_regex=filters_regex)

VibhuJawa

## chunksize = 190k M, 1% of dataset
small_df = df.head(190_000).copy(deep=True)
%time ouptput_df = preprocess_text_df(small_df,filter_regex=filters_regex)

VibhuJawa

author_id = author_name_ls.index('Charles Dickens') 
for index in output_indices_umap[author_id]:
    print(author_name_ls[int(index)])

VibhuJawa

def preprocess_text(input_strs ,filter_regex,stop_words = nltk.corpus.stopwords.words('english')):
    """
        * filter punctuation
        * to_lower
        * remove stop words (from nltk corpus) (taking the most time)
        * remove multiple space with one
        * remove leading and trailing spaces    
    """
    
   

VibhuJawa

author_id = author_name_ls.index('Charles Dickens')
for index in output_indices_umap[author_id]:
    print(author_name_ls[int(index)])