lisanka93

df['ColumnA'] = df[df.columns[1:]].apply(
  lambda x: ' '.join(x.dropna().astype(str)),
  axis=1
)

lisanka93

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

example_sent = "This is a sample sentence, showing off the stop words filtration."
stop_words = set(stopwords.words('english'))
word_tokens = word_tokenize(example_sent)

filtered_sentence = [w for w in word_tokens if not w in stop_words]
print(filtered_sentence)

lisanka93

from nltk.util import ngrams, word_tokenize, bigrams, trigrams

sen = "Dummy sentence to demonstrate bigrams"
nltk_tokens = word_tokenize(sen) #using tokenize from NLKT and not split() because split() does not take into account punctuation

#splitting sentence into bigrams and trigrams
print(list(bigrams(nltk_tokens)))
print(list(trigrams(nltk_tokens)))

#creating a dictionary that shows occurances of n-grams in text

lisanka93

import re

#letters only
raw_text = "this is a test. To demonstrate 2 regex expressions!!"
letters_only_text = re.sub("[^a-zA-Z]", " ", raw_text)

#keep numbers
letnum_text =  re.sub("[^a-zA-Z0-9\s]+", " ",raw_text )

lisanka93

from nltk.stem import WordNetLemmatizer, PorterStemmer, SnowballStemmer

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

word = "considering"

stemmed_word =  stemmer.stem(word)
lemmatised_word = lemmatizer.lemmatize(word)

lisanka93

lisanka93

lisanka93

covid_data = pd.read_csv('covid_vacc_concerns.csv')
covid_data['prep_arg'] = covid_data['arg'].apply(preprocess)

lisanka93

X_train, X_test, y_train, y_test = train_test_split(
    covid_data['prep_arg'],
    covid_data['concern'],
    test_size=0.2,
    random_state=50
)

lisanka93

count_vectorizer = CountVectorizer(binary=True)
#fit training data
training_data = count_vectorizer.fit_transform(X_train)

#transform test data
testing_data = count_vectorizer.transform(X_test)