semantic sim

semantic_sim.py

import torch
from transformers import BertTokenizer, BertModel

tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
model = BertModel.from_pretrained('bert-base-uncased')

def encode(text):
    tokens = tokenizer.tokenize(text)
    tokens = ['[CLS]'] + tokens + ['[SEP]']
    return tokenizer.convert_tokens_to_ids(tokens)

ground_truth = """
I am playing cricket because this is a passion for me.
"""

response_text = """
My passion is the game which called cricket.
"""

ground_truth_ids = encode(ground_truth)
response_ids = encode(response_text)

def extract_embeddings(token_ids):
    tokens_tensor = torch.tensor([token_ids])
    with torch.no_grad():
        outputs = model(tokens_tensor)
        hidden_state = outputs[0]
    return hidden_state[0].mean(dim=0)

ground_truth_emb = extract_embeddings(ground_truth_ids)
response_emb = extract_embeddings(response_ids)

cos = torch.nn.CosineSimilarity(dim=0)
similarity = cos(ground_truth_emb, response_emb).item()
print(similarity)

torch
transformers