VibhuJawa

VibhuJawa

VibhuJawa

VibhuJawa

print("Hello World")

VibhuJawa

def get_non_empty_lines(lines):
    """
        returns non empty lines from a list of lines
    """
    clean_lines = []
    for line in lines:
        str_line = line.strip()
        if str_line:
            clean_lines.append(str_line)            
    return clean_lines

VibhuJawa

# remove the following punctuation/characters from cudf
filters = [ '!', '"', '#', '$', '%', '&', '(', ')', '*', '+', '-', '.', '/',  '\\', ':', ';', '<', '=', '>',
           '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '\~', '\t','\\n',"'",",",'~' , '—']

text_col_sample = df.head(5)
text_col_sample['text'].to_pandas()

text_col_sample['text_clean'] = text_col_sample['text'].str.replace_multi(filters, ' ', regex=False)
text_col_sample['text_clean'] = text_col_sample['text_clean'].str.lower()
text_col_sample['text_clean'].to_pandas()

VibhuJawa

text_col_sample['text_clean'] = text_col_sample['text_clean'].str.lower()
text_col_sample['text_clean'].to_pandas()

VibhuJawa

STOPWORDS = nltk.corpus.stopwords.words('english')
STOPWORDS = nvstrings.to_device(STOPWORDS)

text_col_sample['text_clean'] = nvtext.replace_tokens(text_col_sample['text_clean'].data, STOPWORDS, ' ')
text_col_sample['text_clean'].to_pandas()

VibhuJawa

text_col_sample['text_clean'] = text_col_sample['text_clean'].str.replace(r"\s+", ' ',regex=True)
text_col_sample['text_clean'] = text_col_sample['text_clean'].str.strip(' ')
text_col_sample['text_clean'].to_pandas()

VibhuJawa

STOPWORDS = nltk.corpus.stopwords.words('english')

filters = [ '!', '"', '#', '$', '%', '&', '(', ')', '*', '+', '-', '.', '/',  '\\', ':', ';', '<', '=', '>',
           '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '\~', '\t','\\n',"'",",",'~' , '—']

def preprocess_text(input_strs , filters=None , stopwords=STOPWORDS):
    """
        * filter punctuation
        * to_lower
        * remove stop words (from nltk corpus)