FastText model evaluation using word similarity method

word-similarity.py

import pandas as pd
import fasttext
import numpy as np
from scipy.spatial.distance import cosine
from scipy.stats import pearsonr
from scipy.stats import spearmanr
from scipy.stats import kendalltau

# Load pre-trained FastText model
model = fasttext.load_model('models/cc.en.300.bin')

# Load SimVerb-3500 dataset from CSV file
dataset_path = 'datasets/simverb-3500.csv'
#dataset_path = 'datasets/wordsim-353.csv'
#dataset_path = 'datasets/card-660.csv'
dataset = pd.read_csv(dataset_path)

def cosine_similarity(embedding_1, embedding_2):
    # Calculate the cosine similarity of the two embeddings.
    sim = 1 - cosine(embedding_1, embedding_2)
    return round(sim,3)

# Test the model on SimVerb-3500 dataset
total_pairs = len(dataset)
sim_total = 0
rating_total = 0
sim_array = []
for index, row in dataset.iterrows():
    word1 = row['word1']
    word2 = row['word2']
    rating = row['rating']
    word1_vector = model.get_word_vector(word1)
    word2_vector = model.get_word_vector(word2)

    sim = cosine_similarity(word1_vector, word2_vector)
    sim_array.append(sim*10)

    sim_total = sim_total+sim
    rating_total = rating_total+rating

    print(f"Pair: {word1} - {word2} - {rating}")
    print(f"Cosine Similarity: {sim:.3f}")
    print()

sim_avg = (sim_total/total_pairs)*10
rating_avg = rating_total/total_pairs
print(f"Cosine Similarity: {sim_avg:.3f}")
print(f"Rating Similarity: {rating_avg:.3f}")

pearson, _ = pearsonr(np.array(dataset['rating']), np.array(sim_array))
print('Pearsons correlation: %.3f' % pearson)

spearman, _ = spearmanr(dataset['rating'], sim_array)
print('Spearman correlation: %.3f' % spearman)

kendall, _ = kendalltau(np.array(dataset['rating']), sim_array)
print('Kendall correlation: %.3f' % kendall)