RQI evaluation

temp.py

from openai import OpenAI
from configparser import ConfigParser
import json


config = ConfigParser()
config.read("config.ini")

def read_file(filename: str) -> str:
    with open(filename, 'r', encoding='utf-8') as f:
        content = f.read().strip()
    return content


system_prompt = read_file("prompt/system_prompt.txt")
user_prompt = read_file("prompt/user_prompt.txt")

client = OpenAI()
response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": f"{user_prompt}\n\n{'...'}"},
    ],
    response_format={"type": "json_object"},
    temperature=1,
    max_completion_tokens=2048,
    top_p=1,
    frequency_penalty=0,
    presence_penalty=0,
)

def parse_json_and_extract(data):
    try:
        parsed_data = json.loads(data)

        is_invalid = parsed_data.get('is_invalid')
        reason = parsed_data.get('reason')

        if is_invalid is None or reason is None:
            print("Warning: Missing 'is_invalid' or 'reason' in the data.")
        else:
            print(f"is_invalid: {is_invalid}, reason: {reason}")

        return is_invalid, reason

    except json.JSONDecodeError as e:
        print(f"Error parsing JSON: {e}")
        return None, None

result = response.choices[0].message.content
is_invalid, reason = parse_json_and_extract(result)
print(is_invalid, reason)