consensusllm.js

import OpenAI from "openai";
import fs from "fs";
const culture = [
    "meta-llama/llama-3.1-405b-instruct",
    "openai/gpt-4o",
    "anthropic/claude-3.5-sonnet",
    "qwen/qwen-2-72b-instruct",
    "microsoft/wizardlm-2-8x22b"
];

const openai = new OpenAI({
    apiKey: 'CENSORED',
    baseURL: "https://openrouter.ai/api/v1"
});

const messages = [
    { role: "system", content: "You are an AI that writes out descriptions and examples of reasoning tasks. When generating examples, make them 'out of distribution' - ie. involving concepts or scenarios that would be hard to find online. The examples you provide should not be the 'typical' examples provided for the task." },
]

// Load 3 random tasks for few shot prompting
const tasksToGenerate = fs.readFileSync("generated-topics-final.txt", "utf8").split("\n").sort(() => Math.random() - 0.5);
for (const t of tasksToGenerate) {
    console.log(`Generating task: ${t}`);
    let unbroken = true;
    for (let i = 0; i < 3; i++) {
        console.log(`Iteration ${i + 1}`);
        const genTaskDir = "generated-tasks";
        const fileName = t.toLowerCase().replace(/\s/g, "-");
        if (!fs.existsSync(genTaskDir)) {
            fs.mkdirSync(genTaskDir);
        }
        if (fs.existsSync(`${genTaskDir}/${fileName}.md`)) {
            console.log(`Task ${t} already exists`);
            unbroken = false;
            break;
        }
        const NUM_TASKS = 3;
        const taskFolder = "tasks";

        const filesInFolder = fs.readdirSync(taskFolder).sort(() => Math.random() - 0.5).slice(0, NUM_TASKS);

        const tasks = filesInFolder.map(file => {
            const task = fs.readFileSync(`${taskFolder}/${file}`, "utf8");
            return task;
        })

        const taskNames = tasks.map(task => task.split("\n")[0].replace("#", "").trim());

        for (let i = 0; i < tasks.length; i++) {
            messages.push({ role: "user", content: `Write out a description and examples of the reasoning task: ${taskNames[i]}` });
            messages.push({ role: "assistant", content: tasks[i] });
        }

        const taskToGenerate = t;
        messages.push({ role: "user", content: `Write out a description and examples of the reasoning task: ${taskToGenerate}` });

        const response = await openai.chat.completions.create({
            messages,
            model: "anthropic/claude-3.5-sonnet",
            temperature: 0.7
        });

        fs.writeFileSync("generated.txt", response.choices[0].message.content);


        let generated = fs.readFileSync("generated.txt").toString();
        let examples = [];
        while (true) {
            let start = generated.indexOf("### Example");
            if (start === -1) {
                break;
            }
            start += 4;
            let end = generated.indexOf("---");
            end += 3;
            if (end === 2) {
                end = generated.indexOf("## Tags:");
            }
            let example = generated.substring(start, end).trim();
            examples.push(example);
            generated = generated.substring(end);
        }
        examples = examples.map(example => {
            const inputStart = example.indexOf("Input:");
            const inputEnd = example.indexOf("Output:");
            const input = example.substring(inputStart + 6, inputEnd).trim();
            const output = example.substring(inputEnd + 7).trim();
            return { input, output };
        });

        // Get all the culture model's outputs
        for (const example of examples) {
            console.log(`Processing example #${examples.indexOf(example) + 1}`);
            const cultureResponses = await Promise.all(culture.map(model => {
                return openai.chat.completions.create({
                    messages: [
                        { role: "system", content: "You are a helpful AI  that answers reasoning questions and puzzles." },
                        { role: "user", content: example.input },
                    ],
                    model,
                    temperature: 0.4
                });
            }));
            example.responses = cultureResponses.map(response => response.choices[0].message.content);
        }
        fs.writeFileSync("examples.json", JSON.stringify(examples, null, 4));

        // Load the examples
        const reviews = [];
        for (const example of examples) {
            console.log(`Reviewing example #${examples.indexOf(example) + 1}`);
            const reviewMessage = `
    Here's the question:
    ${example.input}
    Here's the current answer:
    ${example.output}
    Here's answers provided by a variety of other parties:
    ${example.responses.join("\n")}
    `;
            const messages = [
                { role: "system", content: "You are an AI that examines a question, a current answer, and a variety of other answers to determine if the other answers are similar to the current answer. Examine the similarity of the answers between <thinking> and </thinking> XML tags. Then provide one of three ratings - AGREE, MIXED, and CHAOS. AGREE for if the answers are sensible and similar to the current answer, MIXED, if one or two answers are meaningfully different, CHAOS if all the answers say different things, and WRONG if all the other answers are similar but the current answer is the odd one out. Place this rating between <rating> and </rating> XML tags." },
                { role: "user", content: reviewMessage },
            ];
            const response = await openai.chat.completions.create({
                messages,
                model: "anthropic/claude-3.5-sonnet",
                temperature: 0.4
            });
            const review = response.choices[0].message.content;
            reviews.push(review);
        }

        fs.writeFileSync("reviews.txt", reviews.join("\n\n"));

        // Check that all reviews are AGREE

        const allAgree = reviews.every(review => review.includes("AGREE"));

        if (allAgree) {
            console.log("All reviews are AGREE");

            fs.writeFileSync(`${genTaskDir}/${fileName}.md`, fs.readFileSync("generated.txt"));
            unbroken = false;
            break;
        }
    }
    if (unbroken) {
        console.log(`Failed to generate task ${t}`);
    }
}