FastText model evaluation using word analogy method

word-analogy.py

import numpy as np
import fasttext
import csv

# Load FastText word embeddings (pre-trained model)
model = fasttext.load_model('models/cc.en.300.bin')

def word_analogy_3cosadd(model, word1, word2, word3, top_k=1):
    # Get word vectors
    vector1 = model.get_word_vector(word1)
    vector2 = model.get_word_vector(word2)
    vector3 = model.get_word_vector(word3)

    # Compute analogy vector using 3CosAdd
    analogy_vector = vector2 - vector1 + vector3

    # Calculate cosine similarity between analogy vector and all word vectors
    cosine_similarities = {}
    for word in model.get_words():
        vector = model.get_word_vector(word)
        cosine_similarities[word] = np.dot(vector, analogy_vector) / (np.linalg.norm(vector) * np.linalg.norm(analogy_vector))

    # Sort the words based on cosine similarity scores
    sorted_words = sorted(cosine_similarities.items(), key=lambda x: x[1], reverse=True)

    # Return the top-k similar words
    similar_words = [word for word, _ in sorted_words[:top_k]]

    return similar_words

def word_analogy_3cosmul(model, word1, word2, word3, top_k=1):
    # Get word vectors
    vector1 = model.get_word_vector(word1)
    vector2 = model.get_word_vector(word2)
    vector3 = model.get_word_vector(word3)

    # Compute analogy vector using 3CosMul
    analogy_vector = vector2 / np.linalg.norm(vector2) - vector1 / np.linalg.norm(vector1) + vector3 / np.linalg.norm(vector3)

    # Calculate cosine similarity between analogy vector and all word vectors
    cosine_similarities = {}
    for word in model.get_words():
        vector = model.get_word_vector(word)
        cosine_similarities[word] = np.dot(vector, analogy_vector) / (np.linalg.norm(vector) * np.linalg.norm(analogy_vector))

    # Sort the words based on cosine similarity scores
    sorted_words = sorted(cosine_similarities.items(), key=lambda x: x[1], reverse=True)

    # Return the top-k similar words
    similar_words = [word for word, _ in sorted_words[:top_k]]

    return similar_words

# Load MSR analogy dataset from CSV file
dataset_path = 'datasets/msr.csv'
with open(dataset_path, 'r') as file:
    csv_reader = csv.reader(file)
    analogy_dataset = list(csv_reader)

# Evaluate model on word analogy task
cosadd_correct_predictions = 0
cosadd_total_predictions = len(analogy_dataset)
cosmul_correct_predictions = 0
cosmul_total_predictions = len(analogy_dataset)
for analogy in analogy_dataset:
    print(analogy)
    # Test 3CosAdd
    similar_words_3cosadd = word_analogy_3cosadd(model, analogy[2], analogy[3], analogy[4])
    print("3CosAdd:", similar_words_3cosadd)
    #for word in similar_words_3cosadd:
    #    if word == analogy[5]:
    #        cosadd_correct_predictions += 1
    if analogy[5] in similar_words_3cosadd: cosadd_correct_predictions += 1

    # Test 3CosMul
    similar_words_3cosmul = word_analogy_3cosmul(model, analogy[2], analogy[3], analogy[4])
    print("3CosMul:", similar_words_3cosmul)
    for word in similar_words_3cosmul:
        if word == analogy[5]:
            cosmul_correct_predictions += 1
    if analogy[5] in similar_words_3cosmul: cosmul_correct_predictions += 1
# Calculate accuracy
cosadd_accuracy = cosadd_correct_predictions / cosadd_total_predictions
cosmul_accuracy = cosmul_correct_predictions / cosmul_total_predictions

# Print evaluation result
print('Evaluation Results:')
print('3cosadd Number of correct predictions: {}'.format(cosadd_correct_predictions))
print('3cosadd Number of predictions: {}'.format(cosadd_total_predictions))
print('3cosadd Accuracy: {:.2f}'.format(cosadd_accuracy))
print('3cosmul Number of correct predictions: {}'.format(cosmul_correct_predictions))
print('3cosmul Number of predictions: {}'.format(cosmul_total_predictions))
print('3cosmul Accuracy: {:.2f}'.format(cosmul_accuracy))