Stream dialogue in real-time

pip install python-dotenv websockets

npm install dotenv ws
npm install @types/dotenv @types/ws --save-dev

import asyncio
import base64
import json
import os

from dotenv import load_dotenv
import websockets

load_dotenv()
ELEVENLABS_API_KEY = os.getenv("ELEVENLABS_API_KEY")
VOICE_ID = "21m00Tcm4TlvDq8ikWAM"
MODEL_ID = "eleven_v4_turbo"

URI = (
    "wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input"
    f"?model_id={MODEL_ID}&output_format=mp3_44100_128"
)

import * as dotenv from "dotenv";
import * as fs from "node:fs";
import WebSocket from "ws";

dotenv.config();
const ELEVENLABS_API_KEY = process.env.ELEVENLABS_API_KEY;
const voiceId = "21m00Tcm4TlvDq8ikWAM";
const modelId = "eleven_v4_turbo";

const uri = `wss://api.elevenlabs.io/v1/text-to-dialogue/stream-input?model_id=${modelId}&output_format=mp3_44100_128`;
const websocket = new WebSocket(uri);

const outputDir = "./output";
try {
  fs.accessSync(outputDir, fs.constants.R_OK | fs.constants.W_OK);
} catch {
  fs.mkdirSync(outputDir);
}
const writeStream = fs.createWriteStream(`${outputDir}/dialogue-ws.mp3`, { flags: "a" });

async def stream_dialogue():
    async with websockets.connect(URI) as websocket:
        await websocket.send(
            json.dumps(
                {
                    "voices": [VOICE_ID],
                    "xi_api_key": ELEVENLABS_API_KEY,
                }
            )
        )

        line = (
            "This is a longer line of dialogue used to exceed the minimum buffer so the model "
            "starts generating streamed audio for the registered voice. "
        )
        await websocket.send(
            json.dumps(
                {
                    "inputs": [
                        {"text": line, "voice_id": VOICE_ID, "new_turn": False},
                    ],
                }
            )
        )

        await websocket.send(json.dumps({"close_socket": True}))

        os.makedirs("output", exist_ok=True)
        out_path = "output/dialogue-ws.mp3"
        with open(out_path, "wb") as audio_file:
            while True:
                raw = await websocket.recv()
                msg = json.loads(raw)
                if msg.get("error"):
                    raise RuntimeError(msg)
                if msg.get("audio"):
                    audio_file.write(base64.b64decode(msg["audio"]))
                if msg.get("is_final"):
                    break
        print(f"Wrote {out_path}")


asyncio.run(stream_dialogue())

websocket.on("open", () => {
  websocket.send(
    JSON.stringify({
      voices: [voiceId],
      xi_api_key: ELEVENLABS_API_KEY,
    })
  );

  const line =
    "This is a longer line of dialogue used to exceed the minimum buffer so the model starts generating streamed audio for the registered voice. ";

  websocket.send(
    JSON.stringify({
      inputs: [{ text: line, voice_id: voiceId, new_turn: false }],
    })
  );

  websocket.send(JSON.stringify({ close_socket: true }));
});

websocket.on("message", (data) => {
  const msg = JSON.parse(data.toString());
  if (msg.error) {
    console.error(msg);
    return;
  }
  if (msg.audio) {
    writeToLocal(msg.audio, writeStream);
  }
});

function writeToLocal(base64str: string, stream: fs.WriteStream) {
  stream.write(Buffer.from(base64str, "base64"));
}

websocket.on("close", () => {
  writeStream.end();
});

python text-to-dialogue-websocket.py

npx tsx text-to-dialogue-websocket.ts