STREAMING FROM CF WORKER ACCOMPLISHED!!

index.ts

import { ChatOpenAI } from "langchain/chat_models";
import { HumanChatMessage } from "langchain/schema";
import { CallbackManager } from "langchain/callbacks";

interface Env {
  OPENAI_API_KEY: string;
}

export default {
  async fetch(
    _request: Request,
    env: Env,
    ctx: ExecutionContext
  ): Promise<Response> {
    const { readable, writable } = new TransformStream();
    const writer = writable.getWriter();

    ctx.waitUntil(getResponse(writer, env));

    return new Response(readable, {
      status: 200,
      headers: {
        "Content-Type": "text/event-stream",
        "Cache-Control": "no-cache, no-transform",
        Connection: "keep-alive",
      },
    });
  },
};

async function getResponse(
  writer: WritableStreamDefaultWriter,
  env: Env
): Promise<void> {
  const encoder = new TextEncoder();
  writer.write(encoder.encode("[START]\n"));

  const chatStreaming = new ChatOpenAI({
    openAIApiKey: env.OPENAI_API_KEY,
    streaming: true,
    callbackManager: CallbackManager.fromHandlers({
      async handleLLMNewToken(token: string) {
        // Send the token back to the client.
        await writer.write(encoder.encode(token));
      },
    }),
  });

  await chatStreaming.call([
    new HumanChatMessage(
      "Write a song about sparkling water"
    ),
  ]);

  writer.write(encoder.encode("\n[DONE]"));
  return writer.close();
}