Prompt to calculate quality

calculate-quality.ts

// TASK: calculate
// Run this task with:
// forge task:run quality:calculate --uuid 94a2f0b6-66e5-4674-9f3c-ab77acc8b903

import { createTask } from '@forgehive/task'
import { Schema } from '@forgehive/schema'
import { OpenAI } from 'openai'
import { Idea } from '@/models'

const description = 'Load and process an idea by UUID'

const schema = new Schema({
  uuid: Schema.string()
})

const boundaries = {
  findByUuid: async (uuid: string) => {
    const ideaDoc = await Idea.findOne({ uuid })

    if (!ideaDoc) {
      return null
    }

    return ideaDoc.toJSON()
  },
    evaluateBenefitClarity: async (pressReleaseContent: string): Promise<{ benefit: number; reason: string; suggestions: string; usage: number | undefined }> => {
    const apiKey = process.env.OPENAI_API_KEY;

    if (!apiKey) {
      throw new Error('OpenAI API key is not configured. Please set OPENAI_API_KEY in your .env file.');
    }

    const openai = new OpenAI({ apiKey });

    const systemPrompt = `
      You are an expert evaluator of product communication clarity. Your task is to evaluate how clearly the benefit for the user is described in the given press release.

      Rate the benefit clarity on a scale from 0 to 10, where:
      - 0: No user benefits mentioned
      - 1-3: Very unclear, vague, or missing user benefits
      - 4-6: Somewhat clear but could be more specific or compelling
      - 7-8: Clear and well-articulated user benefits
      - 9-10: Exceptionally clear, specific, and compelling user benefits

      Consider factors like:
      - How specific are the benefits mentioned?
      - How clearly is the value proposition communicated?
      - How well does it explain what the user will gain?
      - How compelling and understandable are the benefits?

      Provide your evaluation with:
      1. A numeric score (0-10)
      2. A clear reason explaining why you gave this score
      3. Specific suggestions for improvement

      Respond with a JSON object in this exact format:
      {"benefit": <number>, "reason": "<explanation>", "suggestions": "<improvement recommendations>"}
    `;

    const response = await openai.chat.completions.create({
      model: "gpt-4o-mini-2024-07-18",
      messages: [
        {
          role: "system",
          content: systemPrompt
        },
        {
          role: "user",
          content: pressReleaseContent,
        }
      ],
      max_tokens: 500,
      temperature: 0.3,
    });

    if (!response.choices[0].message.content) {
      throw new Error('Could not evaluate benefit clarity from the press release.');
    }

    const generatedContent = response.choices[0].message.content;

    try {
      const evaluation = JSON.parse(generatedContent);

      if (typeof evaluation.benefit !== 'number' || evaluation.benefit < 0 || evaluation.benefit > 10) {
        throw new Error('Invalid benefit score received from LLM');
      }

      if (typeof evaluation.reason !== 'string' || typeof evaluation.suggestions !== 'string') {
        throw new Error('Invalid evaluation format received from LLM');
      }

      return {
        benefit: evaluation.benefit,
        reason: evaluation.reason,
        suggestions: evaluation.suggestions,
        usage: response.usage?.total_tokens
      };
    } catch (parseError) {
      throw new Error('Could not parse quality evaluation response as JSON');
    }
  },
  saveQuality: async (uuid: string, quality: { benefit: number; reason: string; suggestions: string }) => {
    await Idea.findOneAndUpdate(
      { uuid },
      { quality },
      { new: true }
    );
    return true;
  }
}

export const calculate = createTask(
  schema,
  boundaries,
  async function ({ uuid }, { findByUuid, evaluateBenefitClarity, saveQuality }) {

    const idea = await findByUuid(uuid)

    if (!idea) {
      return {
        status: 'Error',
        message: `Idea with UUID ${uuid} not found`
      }
    }

    // Check if the idea has press release content to evaluate
    if (!idea.content || !idea.title) {
      return {
        status: 'Error',
        message: 'Idea does not have press release content to evaluate. Please generate a press release first.'
      }
    }

          // Combine title and content for evaluation
      const pressReleaseContent = `${idea.title}\n\n${idea.content}`;

      // Evaluate benefit clarity using LLM
      const evaluation = await evaluateBenefitClarity(pressReleaseContent);

      // Save quality score to the idea
      const quality = {
        benefit: evaluation.benefit,
        reason: evaluation.reason,
        suggestions: evaluation.suggestions
      };
      await saveQuality(uuid, quality);

      return {
        status: 'Ok',
        quality,
        usage: evaluation.usage,
        idea: {
          uuid: idea.uuid,
          title: idea.title
        }
      };
  }
)

calculate.setDescription(description)