How to restart live transcription listener after stopping? [Node SDK]

[andrewreece]

Hello DeepGram community! I'm having trouble figuring out how to restart the live transcription websocket after it closes. When I first start up my application, everything is working great and the audio captured from client is successfully transcribed by DeepGram. The transcriptReceived event functions properly and returns transcribed text.

For example, here's the server console readout when I start up an application. (I say "test test test" into my computer microphone, and you can see it gets transcribed correctly.)

Connected on server side with ID: VDmNjYW0EgbzRX3kAAAB
dgLive WEBSOCKET CONNECTION OPEN!
Packet Received from client. DG STATE: 1
Packet Received from client. DG STATE: 1
Packet Received from client. DG STATE: 1
Packet Received from client. DG STATE: 1
Packet Received from client. DG STATE: 1
transcriptReceived listener event
NEW UTTERANCE: test test test.

But I have a couple of use cases where I want to stop the DG listener (such as when the client-side MediaRecorder is paused for a long time). I can close the DG live object with:

dgLiveObj.send(JSON.stringify({ 'type': 'CloseStream' }))

But when I create a new dgLiveObj instance a few moments later (same web server, same client-server websocket), the object seems to be working correctly (it is returning ReadyState === 1). I call the same creation code as I do when I'm first starting up the application:

const createNewDeepgram = () => new Deepgram(process.env.DEEPGRAM_API_KEY);
const createNewDeepgramLive = (dg) => dg.transcription.live({ punctuate: true });

const addDeepgramTranscriptListener = (dg) => {
  dg.addListener('transcriptReceived', async (dgOutput) => {
    console.log('transcriptReceived listener event');
      let dgJSON = JSON.parse(dgOutput);
      let utterance;
      try {
        utterance = dgJSON.channel.alternatives[0].transcript;
      } catch (error) {
        console.log('WARNING: parsing dgJSON failed. Response from dgLive is:');
        console.log(dgJSON);
      }
      if (utterance) console.log(`NEW UTTERANCE: ${utterance}`);
  });
};
deepgram = createNewDeepgram();
dgLiveObj = createNewDeepgramLive(deepgram);
addDeepgramTranscriptListener(dgLiveObj);

I can also see that audio packets are still being sent successfully from client to server - the websocket between my client and my server has a functional open connection.

But nothing comes back from the new transcriptReceived listener anymore, and I don't know why.

I've been trying to troubleshoot this issue for a week now, and I haven't made much progress. One thing that would help me to debug is to know whether the .send(event) method is functioning, and it's the listener that is the problem, or whether the new dgLiveObj is not actually sending audio packets back to the DG server to be transcribed. But really, I don't know and I'm feeling pretty stuck.

I would be very appreciative of any guidance anyone has to offer. Thank you! 🙏

Server code:

import * as dotenv from 'dotenv';
dotenv.config({ path: '~/.env' });

import express from 'express';
import { createServer } from "http";
import { Server } from "socket.io";

const app = express();
app.use(express.static('public/'));
app.get("/", function (req, res) {
    res.sendFile(__dirname + "/dg-test-index.html");
});
const httpServer = createServer(app);

import pkg from '@deepgram/sdk';
const { Deepgram } = pkg;
let deepgram;
let dgLiveObj;
let io;

const createWebsocket = () => {
  io = new Server(httpServer, { transports: "websocket" });
  io.on('connection', (socket) => {
    console.log(`Connected on server side with ID: ${socket.id}`);
    deepgram = createNewDeepgram();
    dgLiveObj = createNewDeepgramLive(deepgram);
    addDeepgramTranscriptListener(dgLiveObj);
    addDeepgramOpenListener(dgLiveObj);
    addDeepgramCloseListener(dgLiveObj);
    addDeepgramErrorListener(dgLiveObj);

    socket.on('dg-close', async (msg) => dgLiveObj.send(JSON.stringify({ 'type': 'CloseStream' })));
    socket.on('dg-open', async (msg, callback) => dgReopen(msg).then((status) => callback(status)));
    socket.on('packet-sent', async (event) => dgPacketResponse(event, dgLiveObj));
  });
};

const createNewDeepgram = () => new Deepgram(process.env.DEEPGRAM_API_KEY);
const createNewDeepgramLive = (dg) => dg.transcription.live({ punctuate: true });

const addDeepgramTranscriptListener = (dg) => {
  dg.addListener('transcriptReceived', async (dgOutput) => {
    console.log('transcriptReceived listener event');
      let dgJSON = JSON.parse(dgOutput);
      let utterance;
      try {
        utterance = dgJSON.channel.alternatives[0].transcript;
      } catch (error) {
        console.log('WARNING: parsing dgJSON failed. Response from dgLive is:');
        console.log(dgJSON);
      }
      if (utterance) console.log(`NEW UTTERANCE: ${utterance}`);
  });
};

const addDeepgramOpenListener = (dg) => {
  dg.addListener('open', async (msg) => console.log(`dgLive WEBSOCKET CONNECTION OPEN!`));
};

const addDeepgramCloseListener = (dg) => {
  dg.addListener('close', async (msg) => console.log(`dgLive CONNECTION CLOSED!`));
};

const addDeepgramErrorListener = (dg) => {
  dg.addListener('error', async (msg) => console.log(`dgLive ERROR::Type:${msg.type} / Code:${msg.code}`));
};

const dgReopen = async (msg) => {
  console.log(`Reopen message is: ${msg.message}`);
  createWebsocket();
  
  return "let's go!";
};

const dgPacketResponse = (event, dg) => {
  console.log(`Packet Received from client. DG STATE: ${dg.getReadyState()}`);
  if (dg.getReadyState() === 1) dg.send(event); 
};

httpServer.listen(3000);
createWebsocket();

Client code:

let socket;
let mediaRecorder;

$('.pause-media-recorder').click( function(e) {
  mediaRecorder.pause();
});

$('.resume-media-recorder').click( function(e) {
  mediaRecorder.resume();
});

$('.state-media-recorder').click( function(e) {
  $('.mr-state-message .message-body').text(mediaRecorder.state);
});


const startStream = () => {
  navigator.mediaDevices.getUserMedia({ audio: true }).then((stream) => {
    mediaRecorder = new MediaRecorder(stream);
    socket = io(options = { transports: ["websocket"] });

    
  }).then(() => {

    socket.on('connect', async () => {
      console.log('CONNECTED');
      console.log(`Socket ID on Client side: ${socket.id}`);
      if (mediaRecorder.state == 'inactive') mediaRecorder.start(500);

      $('.mr-state-message .message-body').text(mediaRecorder.state);

      mediaRecorder.addEventListener('dataavailable', (event) => {
        console.log('SENDING PACKET TO SERVER');
        socket.emit('packet-sent', event.data);
      });  

      $('.send-dg-close').click( function(e) {
        socket.emit('dg-close');
        mediaRecorder.pause();
      });

      $('.send-dg-open').click( function(e) {
        mediaRecorder.pause();
        socket.disconnect();
        socket.connect();
        socket.emit('dg-open', {message: 'dg-open'}, async (response) => console.log(response));
        setTimeout(() => mediaRecorder.resume(), 1000);
      });
    });

  });

};

startStream();

HTML:

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8">
    <title>Deepgram Test</title>
    <script src="https://cdn.socket.io/4.5.3/socket.io.min.js" integrity="sha384-WPFUvHkB1aHA5TDSZi6xtDgkF0wXJcIIxXhC6h8OT8EH3fC5PWro5pWJ1THjcfEi" crossorigin="anonymous"></script>
    <script src="https://kit.fontawesome.com/aabf17dcb4.js" crossorigin="anonymous"></script>
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/css/bulma.min.css">  
    <script src="BulmaJS-0.12.0/dist/bulma.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.1/jquery.min.js"></script>
  </head>
  <body>
    <div class="columns">
      <div class="column is-5">
        
        <p class="box">foo</p>
        <div class="container">
          <a class="button pause-media-recorder">Pause Media Recorder</a>
          <a class="button resume-media-recorder">Resume Media Recorder</a>
          <br />
          <a class="button state-media-recorder">Give Media Recorder state</a>
        </div>
        <br />
        <div class="container">
          <a class="button send-dg-close">Send DG CLOSE Command</a>
          <a class="button send-dg-open">Send DG OPEN Command</a>
        </div>

      </div>
    </div>
    <article class="message mr-state-message">
      <div class="message-header">
        <p>Current Media Recorder state</p>
        <button class="delete" aria-label="delete"></button>
      </div>
      <div class="message-body">bar</div>
    </article>
    <script src="dg-connection-test-client.js"></script>
  </body>
</html>

[briancbarrow]

Hey @andrewreece! I'll take a look and see if I can figure out what's happening. Do you have your HTML code? Or maybe a repo I can look at too?

[andrewreece]

Hey @briancbarrow! Thanks for having a look, much appreciated! I just added HTML code - just a MVE interface to help me test stopping/restarting the Deepgram object. But should be sufficient for troubleshooting. Thank you thank you!

[briancbarrow]

@andrewreece Sorry for the delay. I ended up going to @BrianHillis-DG for another set of eyes on it. This is the solution he came up with.

Server side:

// import * as dotenv from "dotenv";
// dotenv.config({ path: "~/.env" });
// console.log("process", process.env.DEEPGRAM_API_KEY);
import path from "path";
import { fileURLToPath } from "url";

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

import express from "express";
import { createServer } from "http";
import { Server } from "socket.io";

const app = express();
app.use(express.static("public/"));
app.get("/", function (req, res) {
  res.sendFile(__dirname + "/index.html");
});
const httpServer = createServer(app);

import pkg from "@deepgram/sdk";
const { Deepgram } = pkg;
let deepgram;
let dgLiveObj;
let io;
// make socket global so we can access it from anywhere
let globalSocket;

// Pull out connection logic so we can call it outside of the socket connection event
const initDgConnection = (disconnect) => {
  dgLiveObj = createNewDeepgramLive(deepgram);
  addDeepgramTranscriptListener(dgLiveObj);
  addDeepgramOpenListener(dgLiveObj);
  addDeepgramCloseListener(dgLiveObj);
  addDeepgramErrorListener(dgLiveObj);
  // clear event listeners
  if (disconnect) {
    globalSocket.removeAllListeners();
  }

  globalSocket.on("dg-close", async (msg) =>
    dgLiveObj.send(JSON.stringify({ type: "CloseStream" }))
  );
  globalSocket.on("dg-open", async (msg, callback) =>
    dgReopen(msg).then((status) => callback(status))
  );
  globalSocket.on("packet-sent", async (event) =>
    dgPacketResponse(event, dgLiveObj)
  );
};

const createWebsocket = () => {
  io = new Server(httpServer, { transports: "websocket" });
  io.on("connection", (socket) => {
    console.log(`Connected on server side with ID: ${socket.id}`);
    globalSocket = socket;
    deepgram = createNewDeepgram();
    initDgConnection(false);
  });
};

const createNewDeepgram = () => new Deepgram("YOUR_API_KEY");
const createNewDeepgramLive = (dg) =>
  dg.transcription.live({ punctuate: true });

const addDeepgramTranscriptListener = (dg) => {
  console.log("addDeepgramTranscriptListener");
  dg.addListener("transcriptReceived", async (dgOutput) => {
    console.log("transcriptReceived listener event");
    let dgJSON = JSON.parse(dgOutput);
    let utterance;
    try {
      utterance = dgJSON.channel.alternatives[0].transcript;
    } catch (error) {
      console.log("WARNING: parsing dgJSON failed. Response from dgLive is:");
      console.log(dgJSON);
    }
    if (utterance) console.log(`NEW UTTERANCE: ${utterance}`);
  });
};

const addDeepgramOpenListener = (dg) => {
  dg.addListener("open", async (msg) =>
    console.log(`dgLive WEBSOCKET CONNECTION OPEN!`)
  );
};

const addDeepgramCloseListener = (dg) => {
  dg.addListener("close", async (msg) =>
    console.log(`dgLive CONNECTION CLOSED!`)
  );
};

const addDeepgramErrorListener = (dg) => {
  dg.addListener("error", async (msg) =>
    console.log(`dgLive ERROR::Type:${msg.type} / Code:${msg.code}`)
  );
};

const dgReopen = async (msg) => {
  console.log(`Reopen message is: ${msg.message}`);
  initDgConnection(true);

  return "let's go!";
};

const dgPacketResponse = (event, dg) => {
  if (dg.getReadyState() === 1) {
    dg.send(event);
  }
};

httpServer.listen(3000);
createWebsocket();

Client side (I included the client side script here in a script tag):

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8" />
    <title>Deepgram Test</title>
    <script
      src="https://cdn.socket.io/4.5.3/socket.io.min.js"
      integrity="sha384-WPFUvHkB1aHA5TDSZi6xtDgkF0wXJcIIxXhC6h8OT8EH3fC5PWro5pWJ1THjcfEi"
      crossorigin="anonymous"
    ></script>
    <script
      src="https://kit.fontawesome.com/aabf17dcb4.js"
      crossorigin="anonymous"
    ></script>
    <link
      rel="stylesheet"
      href="https://cdn.jsdelivr.net/npm/[email protected]/css/bulma.min.css"
    />
    <!-- <script src="BulmaJS-0.12.0/dist/bulma.js"></script> -->
    <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.1/jquery.min.js"></script>
  </head>
  <body>
    <div class="columns">
      <div class="column is-5">
        <p class="box">foo</p>
        <div class="container">
          <a class="button pause-media-recorder">Pause Media Recorder</a>
          <a class="button resume-media-recorder">Resume Media Recorder</a>
          <br />
          <a class="button state-media-recorder">Give Media Recorder state</a>
        </div>
        <br />
        <div class="container">
          <a class="button send-dg-close">Send DG CLOSE Command</a>
          <a class="button send-dg-open">Send DG OPEN Command</a>
        </div>
      </div>
    </div>
    <article class="message mr-state-message">
      <div class="message-header">
        <p>Current Media Recorder state</p>
        <button class="delete" aria-label="delete"></button>
      </div>
      <div class="message-body">bar</div>
    </article>
    <script>
      let socket;
      let mediaRecorder;

      console.log("STARTING STREAM");
      $(".pause-media-recorder").click(function (e) {
        mediaRecorder.pause();
      });

      $(".resume-media-recorder").click(function (e) {
        mediaRecorder.resume();
      });

      $(".state-media-recorder").click(function (e) {
        $(".mr-state-message .message-body").text(mediaRecorder.state);
      });

      const startStream = () => {
        navigator.mediaDevices
          .getUserMedia({ audio: true })
          .then((stream) => {
            mediaRecorder = new MediaRecorder(stream);
            socket = io((options = { transports: ["websocket"] }));
          })
          .then(() => {
            socket.on("connect", async () => {
              console.log("CONNECTED");
              console.log(`Socket ID on Client side: ${socket.id}`);
              if (mediaRecorder.state == "inactive") mediaRecorder.start(500);

              $(".mr-state-message .message-body").text(mediaRecorder.state);

              mediaRecorder.addEventListener("dataavailable", (event) => {
                console.log("DATA AVAILABLE", event.data);
                console.log("SENDING PACKET TO SERVER");
                socket.emit("packet-sent", event.data);
              });
              $(".send-dg-close").click(function (e) {
                socket.emit("dg-close");
                mediaRecorder.pause();
              });

              $(".send-dg-open").click(function (e) {
                // stop the media recorder instead of just pausing it
                mediaRecorder.stop();
                socket.emit(
                  "dg-open",
                  { message: "dg-open" },
                  async (response) => {
                    mediaRecorder.start(500);
                  }
                );
              });
            });
          });
      };

      startStream();
    </script>
  </body>
</html>

Let us know if you have other questions.

[andrewreece]

Wow, awesome thank you @briancbarrow and @BrianHillis-DG!! Looks like globalSocket.removeAllListeners(); was the key I was missing. Traveling today but will try this out soon and let you know once I've got it up and running.

I'm so appreciative for your support. Thank you very very much!! 🙇

[andrewreece]

Hi - just following up to say this solution worked and completely unblocked me. It would have taken me ages to figure out what wasn't working without your help. Many many thanks to you both.

[BekahHW]

That's great to hear @andrewreece! Can't wait to hear more about your project. Feel free to post about it in our Show and Tell category.