Compare Thow Phrases Similarity

Thow_Phrases_Similarity.py

import os
from git import Repo
from langchain_community.document_loaders.generic import GenericLoader
from langchain_community.document_loaders.parsers import LanguageParser
from langchain_text_splitters import Language
from qdrant_client import models, QdrantClient
import numpy as np
import torch
from sentence_transformers import SentenceTransformer

# Clone
print('Cloning Repository...')
repo_path = "/root/uepython/"
#repo = Repo.clone_from("https://github.com/crashangelarts/uepython", to_path=repo_path)

# Load
print('Loading...')
loader = GenericLoader.from_filesystem(
    repo_path + "/libs/core/langchain_core",
    glob="**/*",
    suffixes=[".py"],
    exclude=["**/non-utf8-encoding.py"],
    parser=LanguageParser(language=Language.PYTHON, parser_threshold=500),
)
documents = loader.load()
print(f'Loaded {len(documents)} documents')

# Split
print('Splitting...')
from langchain_text_splitters import RecursiveCharacterTextSplitter

python_splitter = RecursiveCharacterTextSplitter.from_language(
    language=Language.PYTHON, chunk_size=2000, chunk_overlap=200
)
texts = python_splitter.split_documents(documents)
print(f'Split {len(texts)} texts')

# Create Qdrant client
print('Connecting to Qdrant...')
qdrant = QdrantClient(":memory:")
encoder = SentenceTransformer('all-MiniLM-L6-v2')

# Upload embeddings and get similarity
def upload_embd_get_similarity(user_ans, gpt_ans):
    qdrant.recreate_collection(
        collection_name="m",
        vectors_config=models.VectorParams(
            size=encoder.get_sentence_embedding_dimension(),
            distance=models.Distance.COSINE
        )
    )

    qdrant.upload_records(
        collection_name="m",
        records=[
            models.Record(
                id=1,
                vector=encoder.encode(user_ans).tolist(),
                payload={'text': user_ans}
            )
        ]
    )

    hits = qdrant.search(
        collection_name="m",
        query_vector=encoder.encode(gpt_ans).tolist(),
        limit=1
    )

    for hit in hits:
        return hit.score

# Demonstration
user_answer = "This is a demonstration of the code"
gpt_answer = "This is a generated answer"

similarity = upload_embd_get_similarity(user_answer, gpt_answer)
print(f'Similarity: {similarity}')

print('END')