goran-perbelle

index.js

import Fastify from "fastify";
import WebSocket from "ws";
import fs from "fs";
import dotenv from "dotenv";
import fastifyFormBody from "@fastify/formbody";
import fastifyWs from "@fastify/websocket";
// Load environment variables from .env file
dotenv.config();
// Retrieve the OpenAI API key from environment variables. You must have OpenAI Realtime API access.
const { OPENAI_API_KEY } = process.env;
if (!OPENAI_API_KEY) {
	console.error("Missing OpenAI API key. Please set it in the .env file.");
	process.exit(1);
}
// Initialize Fastify
const fastify = Fastify();
fastify.register(fastifyFormBody);
fastify.register(fastifyWs);

// Constants
const SYSTEM_MESSAGE = `You are a helpful assistant for Perbelle Cosmetics, an e-commerce website selling high-quality cosmetics. 

IMPORTANT: Always start the conversation by greeting the customer warmly and introducing yourself as an assistant for Perbelle Cosmetics.

When handling order inquiries, follow these steps precisely:
1. Ask the customer for their order number.
2. Once provided, repeat the order number back to the customer and ask them to confirm if it's correct.
3. If correct, inform the customer that a representative will call them back with information about their shipment.
4. Ask for a preferred callback number.

For other inquiries:
- Always be polite and professional.
- If you don't know something, it's okay to say so.
- You can direct customers to our website perbellecosmetics.com for more information or to make a purchase.

Remember, do not provide specific tracking information over the call. Always follow the process of asking for the order number, confirming it, and promising a callback.`;

const VOICE = "alloy";
const PORT = process.env.PORT || 5050; // Allow dynamic port assignment
// List of Event Types to log to the console. See OpenAI Realtime API Documentation. (session.updated is handled separately.)
const LOG_EVENT_TYPES = [
	"response.content.done",
	"rate_limits.updated",
	"response.done",
	"input_audio_buffer.committed",
	"input_audio_buffer.speech_stopped",
	"input_audio_buffer.speech_started",
	"session.created",
];

function sendInitialMessage(openAiWs) {
	const introMessage =
		"Hello! Thanks for calling, this is Susan from Perbelle Cosmetics, how may I assist you today?";

	// Step 1: Send `conversation.item.create` as an user message
	const userIntroMessage = {
		type: "conversation.item.create",
		item: {
			type: "message",
			role: "user", // Correct role for the assistant’s introductory message
			content: [
				{
					type: "input_text",
					text: `Greet the user with "${introMessage}"`,
				},
			],
		},
	};
	if (openAiWs.readyState === WebSocket.OPEN) {
		openAiWs.send(JSON.stringify(userIntroMessage));
		console.log("Sent initial message to OpenAI");
		openAiWs.send(JSON.stringify({ type: "response.create" }));
		console.log(
			`Triggered response.create to play the assistant’s introductory message in audio for CallSid.`
		);
	} else {
		console.error(
			"OpenAI WebSocket is not open. Cannot send initial message."
		);
	}
}

/// GORAN END

// Root Route
fastify.get("/", async (request, reply) => {
	reply.send({ message: "Twilio Media Stream Server is running!" });
});
// Route for Twilio to handle incoming and outgoing calls
// <Say> punctuation to improve text-to-speech translation
fastify.all("/incoming-call", async (request, reply) => {
	const twimlResponse = `<?xml version="1.0" encoding="UTF-8"?>
                          <Response>
                              <Say>Please hold while we connect you to one of our support agents.</Say>
                              <Pause length="1"/>
                              <Connect>
                                  <Stream url="wss://${request.headers.host}/media-stream" />
                              </Connect>
                          </Response>`;
	reply.type("text/xml").send(twimlResponse);
});

// WebSocket route for media-stream
fastify.register(async (fastify) => {
	fastify.get("/media-stream", { websocket: true }, (connection, req) => {
		console.log("Client connected");
		const openAiWs = new WebSocket(
			"wss://api.openai.com/v1/realtime?model=gpt-4o-mini-realtime-preview",
			{
				headers: {
					Authorization: `Bearer ${OPENAI_API_KEY}`,
					"OpenAI-Beta": "realtime=v1",
				},
			}
		);
		let streamSid = null;
		const sendSessionUpdate = () => {
			const sessionUpdate = {
				type: "session.update",
				session: {
					turn_detection: { type: "server_vad" },
					input_audio_format: "g711_ulaw",
					output_audio_format: "g711_ulaw",
					voice: VOICE,
					instructions: SYSTEM_MESSAGE,
					modalities: ["text", "audio"],
					// functions: functions,
					temperature: 0.8,
				},
			};
			console.log(
				"Sending session update:",
				JSON.stringify(sessionUpdate)
			);
			openAiWs.send(JSON.stringify(sessionUpdate));
		};
		// Open event for OpenAI WebSocket
		openAiWs.on("open", () => {
			console.log("Connected to the OpenAI Realtime API");
			setTimeout(() => {
				sendSessionUpdate();
			}, 250);
		});
		// Listen for messages from the OpenAI WebSocket (and send to Twilio if necessary)
		openAiWs.on("message", (data) => {
			try {
				const response = JSON.parse(data);
				// if (response.type === "session.updated") {
				// 	console.log("Session updated successfully:", response);
				// 	sendInitialMessage(openAiWs);
				// }
				if (response.type === "response.content.text") {
					console.log("AI text response:", response.text);
					// Convert this text to speech and send to Twilio
					const textToSpeech = {
						event: "media",
						streamSid: streamSid,
						media: {
							payload: Buffer.from(response.text).toString(
								"base64"
							),
						},
					};
					connection.send(JSON.stringify(textToSpeech));
				}
				// if (response.type === "function_call") {
				// 	let functionResponse;
				// 	switch (response.function.name) {
				// 		case "get_product_info":
				// 			functionResponse = getProductInfo(
				// 				response.function.arguments.productName
				// 			);
				// 			break;
				// 		case "get_tracking_info":
				// 			functionResponse = getTrackingInfo(
				// 				response.function.arguments.orderNumber
				// 			);
				// 			break;
				// 		case "get_shipping_info":
				// 			functionResponse = getShippingInfo(
				// 				response.function.arguments.location
				// 			);
				// 			break;
				// 		case "get_return_policy":
				// 			functionResponse = getReturnPolicy(
				// 				response.function.arguments.location
				// 			);
				// 			break;
				// 	}
				// 	openAiWs.send(
				// 		JSON.stringify({
				// 			type: "function_call.response",
				// 			id: response.id,
				// 			response: functionResponse,
				// 		})
				// 	);
				// }
				if (LOG_EVENT_TYPES.includes(response.type)) {
					console.log(
						`Received event: ${response.type}`,
						JSON.stringify(response)
					);
				}
				if (response.type === "session.updated") {
					console.log("Session updated successfully:", response);
					sendInitialMessage(openAiWs);
					//					setTimeout(sendInitialMessage, 250, openAiWs); // Send initial message 250ms after session update
				}
				if (
					response.type === "response.audio.delta" &&
					response.delta
				) {
					const audioDelta = {
						event: "media",
						streamSid: streamSid,
						media: {
							payload: Buffer.from(
								response.delta,
								"base64"
							).toString("base64"),
						},
					};
					connection.send(JSON.stringify(audioDelta));
				}
			} catch (error) {
				console.error(
					"Error processing OpenAI message:",
					error,
					"Raw message:",
					data
				);
			}
		});
		// Handle incoming messages from Twilio
		connection.on("message", (message) => {
			try {
				const data = JSON.parse(message);
				switch (data.event) {
					case "media":
						if (openAiWs.readyState === WebSocket.OPEN) {
							const audioAppend = {
								type: "input_audio_buffer.append",
								audio: data.media.payload,
							};
							openAiWs.send(JSON.stringify(audioAppend));
						}
						break;
					case "start":
						streamSid = data.start.streamSid;
						console.log("Incoming stream has started", streamSid);
						break;
					default:
						console.log("Received non-media event:", data.event);
						break;
				}
			} catch (error) {
				console.error(
					"Error parsing message:",
					error,
					"Message:",
					message
				);
			}
		});
		// Handle connection close
		connection.on("close", () => {
			if (openAiWs.readyState === WebSocket.OPEN) openAiWs.close();
			console.log("Client disconnected.");
		});
		// Handle WebSocket close and errors
		openAiWs.on("close", () => {
			console.log("Disconnected from the OpenAI Realtime API");
		});
		openAiWs.on("error", (error) => {
			console.error("Error in the OpenAI WebSocket:", error);
		});
	});
});

fastify.listen({ port: PORT }, (err) => {
	if (err) {
		console.error(err);
		process.exit(1);
	}
	console.log(`Server is listening on port ${PORT}`);
});