Test GLHF API

glhf-chat.ts

import "dotenv/config";
import { read } from "read";
import OpenAI from "openai";
import { Command } from "@commander-js/extra-typings";

const program = new Command()
  .argument("<model-repo>")
  .action(async (repo) => {
    const baseURL = "https://glhf.chat/api/openai/v1";
    const apiKey = process.env.GLHF_API_KEY;
    const openai = new OpenAI({ baseURL, apiKey });

    const memory: Array<OpenAI.Chat.ChatCompletionMessageParam> = [];
    while(true) {
      const text = await getInput();
      memory.push({
        role: "user",
        content: text,
      });

      const stream = await openai.chat.completions.create({
        model: `hf:${repo}`,
        messages: memory,
        stream: true,
      });

      const pieces: string[] = [];
      for await(const chunk of stream) {
        const curr = chunk.choices[0].delta.content || "";
        process.stdout.write(curr);
        pieces.push(curr);
      }

      memory.push({
        role: "assistant",
        content: pieces.join(""),
      });
    }
  });

async function getInput() {
  console.log("");
  return read({ prompt: "> " });
}

program.parse();