ElevenLabs streaming implementation - Typescript

streaming.ts

const voiceId = '' // Pick any voice ID from https://docs.elevenlabs.io/api-reference/voices
const model = 'eleven_monolingual_v1'
const elUrl = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream?optimize_streaming_latency=3` // Optimize for latency
const codec = 'audio/mpeg'
const maxBufferDuration = 60 // Maximum buffer duration in seconds
const maxConcurrentRequests = 3 // Maximum concurrent requests allowed

// Create a new MediaSource and Audio element
const mediaSource = new MediaSource()
const audioElement = new Audio()

// Request Configuration
const request = {
	text: '',
	model_id: model,
	voice_settings: {
		similarity_boost: 0.5,
		stability: 0.35
	}
}

// Queue for managing concurrent requests
const requestQueue: Function[] = []

async function stream(text: string) {
	request.text = text

	// Set up the MediaSource as the audio element's source
	audioElement.src = URL.createObjectURL(mediaSource)

	// Start playing the audio element immediately
	audioElement.play()

	mediaSource.addEventListener('sourceopen', () => {
		const sourceBuffer = mediaSource.addSourceBuffer(codec) // Adjust the MIME type accordingly

		let isAppending = false
		let appendQueue: ArrayBuffer[] = []

		function processAppendQueue() {
			if (!isAppending && appendQueue.length > 0) {
				isAppending = true
				const chunk = appendQueue.shift()
				chunk && sourceBuffer.appendBuffer(chunk)
			}
		}

		sourceBuffer.addEventListener('updateend', () => {
			isAppending = false
			processAppendQueue()
		})

		function appendChunk(chunk: ArrayBuffer) {
			appendQueue.push(chunk)
			processAppendQueue()

			while (mediaSource.duration - mediaSource.currentTime > maxBufferDuration) {
				const removeEnd = mediaSource.currentTime - maxBufferDuration
				sourceBuffer.remove(0, removeEnd)
			}
		}

		async function fetchAndAppendChunks() {
			try {
				// Check if the maximum concurrent requests limit is reached
				if (requestQueue.length >= maxConcurrentRequests) {
					// Queue the request for later execution
					return new Promise((resolve) => {
						requestQueue.push(resolve)
					})
				}

				// Fetch a chunk of audio data
				const response = await fetch(elUrl, {
					method: 'POST',
					headers: {
						Accept: codec,
						'Content-Type': 'application/json',
						'xi-api-key': YOUR_API_KEY_HERE // Put in your own API key
					},
					body: JSON.stringify(request)
				})

				if (!response.body) {
					// Streaming is not supported in this response, handle appropriately
					console.error('Streaming not supported by the server')
					return
				}

				const reader = response.body.getReader()

				while (true) {
					const { done, value } = await reader.read()

					if (done) {
						break // No more data to read
					}

					// Append the received chunk to the buffer
					appendChunk(value.buffer)
				}
			} catch (error) {
				console.error('Error fetching and appending chunks:', error)
			} finally {
				// Remove the request from the queue
				const nextRequest = requestQueue.shift()
				if (nextRequest) {
					nextRequest()
				}
			}
		}

		// Call the function to start fetching and appending audio chunks
		fetchAndAppendChunks()
	})
}