spotting_hallucination.py

import numpy as np
import pandas as pd
import itertools
import seaborn as sns
import matplotlib.pyplot as plt
import openai
import os
import time

from dotenv import load_dotenv
from collections import defaultdict
from sklearn.metrics.pairwise import cosine_similarity

load_dotenv()
openai.api_key = os.environ.get("OPENAI_API_KEY")
EMBEDDINGS_DICT = {}

hallucination_answers = ["Sure, the first thing you need to do is gather all the data related to the stock market because stock fluctuations may impact the fraudulent transaction. Then write a poem about your feelings regarding this fraudulent transaction, this helps to keep your emotions in check. Following this, you may consider contacting the person who committed the fraud, they might change their mind and return the money.",
"It’s important to contact authorities about possible extraterrestrials that may be involved in the fraudulent activity. Analyze the patterns of the planets, sometimes they align in such a way that increases the chances of fraudulent activity.",
"Before contacting your bank, make sure to solve three crossword puzzles. This sharpens your mind and helps you communicate your concerns more effectively. If the fraudulent transaction was done on a Tuesday, wear a blue shirt while reporting it.",
"In order to tackle the fraudulent transaction, choreograph a dance routine that symbolizes the fight against fraud. Presenting this routine to the bank can positively influence their efforts in resolving your case.",
"Firstly, you will need to contact the ghost of Benjamin Franklin - he was a great adversary to fraud in his time. Using a Ouija board is the preferred method.",
"Cook a meal using thyme and rosemary, these herbs are known to deter fraudulent transactions in certain cultures. Have a well-set dinner table discussion about the fraudulent transactions with your family.",
"Try speaking with your pet about the fraudulent transaction. Animals have a higher intuition for noticing red flags that we humans may overlook. Then take a walk in nature to reflect on the transaction before calling your bank.",
"Carefully package the credit card that was used for the fraudulent transaction and send it to the moon. Only then, you can be assured the fraud will not repeat.",
"An important step is to dress in medieval armor - it symbolizes your fight against fraudulent transactions. This will help you gain the confidence to deal with the issue.",
 "Sing a catchy tune about your fraudulent transaction. It's a unique way of seeking support. You can even record the song and send it to your bank."]

valid_answers = ["Certainly, the first thing you should do is to identify and confirm the suspicious transaction on your account. Then, contact your bank immediately through their dedicated helpline for fraudulent transactions. The bank would guide you through necessary measures to secure your account.",
"Sure, start by noting down the details of the fraudulent transaction and then call your bank's customer service line. You’ll be guided on how to report and refute the transaction, and they will also provide instructions to prevent further fraud.",
"Of course, first ensure you have all details of the fraudulent transaction readily available. Subsequently, get in touch with your bank's customer support to inform them about the incident and seek assistance on the next steps.",
"Absolutely, when you detect a potential fraudulent transaction, review your account details carefully. Thereafter, notify your bank immediately using their secure customer service number and follow their directives as per protocol.",
"Indeed, you should first verify the suspicious transactions in your account statement. Afterwards, promptly reach out to your bank’s customer service to report the discrepancy and they will provide guidance on further actions.",
"No problem, initially go over your account and confirm the questionable transaction. Following this, make sure you get in touch with your bank's fraud department as soon as possible for assistance on the matter.",
"Yes, your first course of action should be to vet your account for any unauthorized transactions. Upon confirmation, contact your bank's fraud support hotline and follow the instructions given to safeguard your account.",
"Surely, first you need to scrutinize the suspicious transaction in your account history. Then, promptly communicate with your bank through their support services to dispute the transaction and protect your account.",
"Of course, the first thing is to establish the fraudulent transaction linked to your account. As soon as you do, contact your bank immediately via their fraud reporting line for help on how to proceed.",
"Yes, initially verify the dubious transaction within your bank statement. Then, immediately connect with your bank's customer service and they will instruct you on how to dispute the transaction and secure your account."]

luis_moneda = ['Luis Moneda is mainly known for his work as a musician, producer and DJ. He is recognized in the electronic music and hip hop scene, with several collaborations with renowned artists. Some of his most popular songs include "Sabor a Miel" and "No Me Compares." Furthermore, Luis Moneda is also considered a social activist, supporting causes such as gender equality and LGBTQ+ rights.', 'Luis Moneda is known for being a renowned economist and professor emeritus of economics at the University of Chicago. He is famous for his contributions in the areas of macroeconomics, monetary policy and financial theory. Furthermore, he was Chairman of the United States Federal Reserve from 2010 to 2014 during a crucial period following the 2008 global financial crisis. Luis Moneda has also written several popular articles and books on economics and finance.', 'Luis Moneda is mainly known as a successful businessman and entrepreneur. His area of activity is in the financial industry, where he founded several technology companies focused on innovative financial solutions. He is also recognized for his social work, being a philanthropist dedicated to humanitarian causes and education programs.', 'Luis Moneda is known for his work as an economist and financial consultant. He is recognized for his expertise in investments, market analysis and financial advice. Furthermore, he is also known for sharing his economic knowledge through lectures, books and articles on personal finance and investing.', 'Luis Moneda is known for being a renowned economist and monetary policy expert. His research and contributions are widely recognized in the financial field, especially in the study and analysis of currencies and economic policy.', 'Luis Moneda, also known as Lucho Moneda, is a famous Colombian singer and songwriter. He rose to prominence in Latin music for his catchy lyrics and unique voice. His songs mix different musical genres, such as reggaeton, pop and urban music. In addition to his successful discography, he is also known for his charismatic stage presence and energetic performances.', 'Luis Moneda is known for being a renowned economist and finance expert. He gained prominence for his work in the financial sector, as a consultant and speaker on topics related to economics, investments and financial management.', 'Luis Moneda is known for being a Spanish musician and singer, best known as the lead singer of the Latin pop band Van Gogh\'s La Oreja. The band is internationally recognized for its infectious songs and captivating lyrics.', 'Luis Moneda is known for being an economist and loudmouth. He gained fame through his economic analyses, participation in debates and media interviews. His captivating personality and ability to explain complex concepts in a simple way have made him recognized within the field of economics. However, he can also be a controversial figure due to some of his controversial or extremist views on certain economic issues.', 'Luis Moneda is best known for being the leader of the legendary Mexican rock band "Molotov". He is the lead singer and guitarist of the band, which has gained international prominence with its unique style of fusion of musical genres, provocative lyrics and energetic performances.']

marcel_proust = ['Marcel Proust is known for being one of the most important French writers of the 20th century. He became especially famous for his work entitled "In Search of Lost Time", also called "À la recherche du temps perdu". It is a series of seven books that portrays the protagonist\'s life and reflections in detail, covering themes such as memory, time, love, art and society. Proust is considered one of the great masters of modern literature and his work is admired for its philosophical depth and the richness of his prose.', 'Marcel Proust is known mainly for being a renowned French writer of the 20th century, famous for his masterpiece titled "In Search of Lost Time" ("À la recherche du temps perdu" in French). This is one of the greatest literary works ever written, where Proust explores themes such as memory, love and identity. Proust\'s writing is marked by its psychological depth, poetic style, and meticulous immersion in the flow of thoughts and sensations of his characters. He is considered one of the great names in modern literature.', 'Marcel Proust became known as one of the most important French writers of the 20th century. He is famous for his great work "In Search of Lost Time" (also known as "À La Recherche du Temps Perdu"), which is considered one of the greatest literary achievements of all time. Proust was a renowned novelist and essayist, known for his deep psychological analysis of characters and his exquisite and poetic prose. His work, also characterized by themes of memory, the passage of time and the search for identity, continues to be studied and appreciated internationally.', 'Marcel Proust is known for his great contribution to modernist literature, especially for his work -press "In Search of Lost Time" (À la recherche du temps perdu). He is widely acclaimed for his vivid depiction of complex characters and detailed settings, as well as his innovative use of memory and stream of consciousness in storytelling. His introspective and reflective writing also made him one of the main representatives of the psychological novel.', 'Marcel Proust is known as a renowned French writer of the 20th century, famous for his masterpiece "In Search of Lost Time" (À la recherche du temps perdu). His detailed and introspective writing explores themes such as memory, love, time and the French aristocratic society of the Belle Époque. Proust\'s work is considered one of the most important contributions to modern literature.', 'Marcel Proust is known primarily as an important French writer of the 20th century. His most famous work is called "In Search of Lost Time", a major review of the intricacies of memory and the complexities of social, psychological and artistic life in bourgeois Western society. Proust\'s work is highly valued for its psychological depth, ornate literary style, and meticulous exploration of the dimensions of time, memory, desire, and identity. He is often considered one of the greatest writers of modern literature.', 'Marcel Proust is known as one of the greatest French writers of the 20th century. He is mainly famous for his masterpiece "In Search of Lost Time" (À la recherche du temps perdu), an extensive and complex series of novels that addresses themes such as memory, time, love and subjectivity. The work is considered one of the most influential in modern literature. Furthermore, Proust is recognized for his ability to explore the deep aspects of human psychology and for his rich and poetic prose.', 'Marcel Proust is known as a renowned French writer of the 20th century, mainly for his masterpiece "In Search of Lost Time". This monumental series of autobiographical novels marks an innovation in literature by dealing with time, memory and the human condition in a profound and introspective way. Proust is also recognized for his complex prose and unique literary style, which established him as one of the great influencers of modern literature.', 'Marcel Proust is mainly known for his literary work "In Search of Lost Time" (or "À La Recherche du Temps Perdu"). This work is considered one of the greatest achievements of 20th century literature and is characterized by its length, sophisticated style and detailed examination of memory, love and the search for the meaning of life. Proust is also recognized for his striking use of innovative narrative techniques, which include stream of consciousness and deep character introspection. His writing significantly influenced modern literature.',' Marcel Proust is known as a renowned French writer, famous for his masterpiece "À la recherche du temps perdu" (In Search of Lost Time). The novel, which consists of seven volumes and more than three thousand pages, is considered one of the greatest works of world literature. Proust is acclaimed for his exquisite, detailed and introspective prose, and is recognized for exploring themes such as memory, time and the search for the meaning of life. He is considered one of the great masters of 20th century literature.']

def generate_embedding_dict(embeddings_dict, list_of_strings, model="text-embedding-ada-002"):
    for text in list_of_strings:
        time.sleep(3)
        embeddings_dict[text] = openai.Embedding.create(input = [text], model=model)['data'][0]['embedding']

    return embeddings_dict

def arg_max_choices_similarity_variance(choices, delay=True):

    # Create a list of embeddings
    choices_embeddings = [EMBEDDINGS_DICT[text] for text in choices]
    # Calculate cosine similarity between all pairs of embeddings
    similarity_matrix = cosine_similarity(choices_embeddings)
    # Average the similarity by choice
    similarity_matrix_avg = np.mean(similarity_matrix, axis=0)
    # Select the item with the highest similarity
    maximum_similarity = np.argmax(similarity_matrix_avg)
    # Flatten the similarity matrix into a 1D array
    similarity_values = similarity_matrix.flatten()
    # Calculate the variance of the similarity values
    variance = np.var(similarity_values)

    return variance, maximum_similarity

def permute_answers_pairs(hallucinations, valids, min_size, max_size):
    results_list = []

    for size in range(min_size, max_size + 1):
        for p in itertools.combinations(hallucinations + valids, size):
            targets = [0 if s in hallucinations else 1 for s in p]
            num_hallucinations = sum(1 for s in p if s in hallucinations)
            proportion = num_hallucinations / len(p)
            variance, maximum_similarity = arg_max_choices_similarity_variance(list(p))

            results_list.append({
                "proportion": proportion,
                "acc": targets[maximum_similarity],
                "length": len(p),
                "var": variance
            })

    return results_list
  
  
generate_embedding_dict(EMBEDDINGS_DICT, hallucination_answers)
generate_embedding_dict(EMBEDDINGS_DICT, valid_answers)
results_list = permute_answers_pairs(hallucination_answers, valid_answers, min_size=3, max_size=10)
results_df = pd.DataFrame(results_list)

lengths = sorted(results_df['length'].unique())
colormap = sns.light_palette("green", as_cmap=True)
colors = colormap(np.linspace(0, 1, len(lengths)))
colors = colormap(np.linspace(0.3, 1, len(lengths)))

plt.figure(figsize=(10, 6))
sns.lineplot(x='proportion', y='var', hue='length', data=results_df, errorbar=("ci", 95), palette=list(colors))
plt.title('Average Similarity Variance by Proportion of hallucinating answers')
plt.xlabel("Proportion of hallucinating answers", fontsize=12)
plt.ylabel("Average Similarity Variance", fontsize=12)
plt.legend(title="Size of the answers' set")
plt.show()

df_grouped = results_df.groupby(['proportion', 'length'])['acc'].mean().reset_index()
plt.figure(figsize=(10, 6))
sns.lineplot(x='proportion', y='acc', hue='length', palette=list(colors), data=df_grouped)
plt.xlabel("Proportion of hallucinating answers", fontsize=12)
plt.ylabel("Accuracy", fontsize=12)
plt.title("Accuracy for picking the answer with the highest average similarity", fontsize=14)
plt.legend().set_title("Size of the answers' set")
plt.show()

# The "real" case
generate_embedding_dict(EMBEDDINGS_DICT, luis_moneda)
generate_embedding_dict(EMBEDDINGS_DICT, marcel_proust)

lm_var, lm_argmax = arg_max_choices_similarity_variance(luis_moneda)
mp_var, mp_argmax = arg_max_choices_similarity_variance(marcel_proust)

colors = colormap(np.linspace(0.3, 1, len(lengths)))[::-1]

plt.figure(figsize=(10, 6))
sns.lineplot(x='proportion', y='var', hue='length', data=results_df[results_df.length == 10], errorbar=("ci", 95), palette=list(colors))
plt.scatter([0, 1], [mp_var, lm_var], color='red')
plt.axhline(y=lm_var, color='orange', label="Hallucination", linestyle="dashed")
plt.axhline(y=mp_var, color='purple', label="Non-hallucination", linestyle="dashed")
plt.title('Average Similarity Variance by Proportion of hallucinating answers')
plt.xlabel("Proportion of hallucinating answers", fontsize=12)
plt.ylabel("Average Similarity Variance", fontsize=12)
plt.legend(title="Size of the answers' set", loc="upper left")

plt.show()