Snippet from SecureLlama and BinaNutrisi project

generate.js

export const dynamic = "force-dynamic";

export default async function handler(req, res) {
  if (req.method === "POST") {
    res.setHeader("Content-Type", "text/event-stream");
    res.setHeader("Cache-Control", "no-cache");

    const { prompt } = req.body;

    const completion = await llama.chat.completions.create(
      {
        model: "meta/llama3-70b-instruct",
        messages: [
          {
            role: "system",
            content:
              "add [END] as new line for every end of response in one chunk. You are a chatbot that guides people to helps people determine suitable nutrition and helps people solve the problem of stunting. you provide education about nutrition and stunting. if people ask things that are not related to nutrition or stunting, then that is outside your job. You speak with bahasa Indonesia. add [END] as new line for every end of response in one chunk.",
          },
          { role: "user", content: prompt },
        ],
        temperature: 0.2,
        top_p: 0.7,
        max_tokens: 1024,
        stream: true,
      },
      { responseType: "stream" }
    );

    // let chatResponse = "";

    for await (const chunk of completion) {
      //   chatResponse += chunk.choices[0]?.delta?.content || "";
      res.write(chunk.choices[0]?.delta?.content || "");

      if (chunk.choices[0]?.delta?.content !== undefined) {
        if (
          chunk.choices[0]?.delta?.content.includes("END") ||
          chunk.choices[0]?.delta?.content === "\n" ||
          chunk.choices[0]?.delta?.content === ""
        ) {
          res.end();
          return;
        }
      }
    }

    req.on("close", () => {
      res.end();
    });
  } else {
    res.status(405).json({ error: "Method not allowed." });
  }
}