Whisper live ts

pom.xml

@@ -188,7 +188,7 @@
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>libjitsi</artifactId>
-      <version>1.1-17-g983122e8</version>
+      <version>1.1-23-g86401561</version>
     </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>

src/main/java/org/jitsi/jigasi/TranscriptionGateway.java

@@ -89,17 +89,17 @@ public TranscriptionGatewaySession createOutgoingCall(CallContext ctx)
                 .getString(
                     CUSTOM_TRANSCRIPTION_SERVICE_PROP,
                     null);
-        TranscriptionService service = null;
+        AbstractTranscriptionService service = null;
         if (customTranscriptionServiceClass != null)
         {
             try
             {
-                service = (TranscriptionService)Class.forName(
-                    customTranscriptionServiceClass).getDeclaredConstructor().newInstance();
+                service = (AbstractTranscriptionService)Class.forName(
+                        customTranscriptionServiceClass).getDeclaredConstructor().newInstance();
             }
             catch(Exception e)
             {
-                logger.error("Cannot instantiate custom transcription service");
+                logger.error("Cannot instantiate custom transcription service", e);
             }
         }
 

src/main/java/org/jitsi/jigasi/TranscriptionGatewaySession.java

@@ -68,7 +68,7 @@ public class TranscriptionGatewaySession
     /**
      * The TranscriptionService used by this session
      */
-    private TranscriptionService service;
+    private AbstractTranscriptionService service;
 
     /**
      * The TranscriptHandler which enables publishing a {@link Transcript} and
@@ -114,7 +114,7 @@ public class TranscriptionGatewaySession
      */
     public TranscriptionGatewaySession(AbstractGateway gateway,
                                        CallContext context,
-                                       TranscriptionService service,
+                                       AbstractTranscriptionService service,
                                        TranscriptHandler handler)
     {
         super(gateway, context);

src/main/java/org/jitsi/jigasi/health/SipHealthPeriodicChecker.java

@@ -21,6 +21,7 @@
 import net.java.sip.communicator.service.protocol.event.*;
 import net.java.sip.communicator.service.protocol.media.*;
 import org.apache.commons.lang3.StringUtils;
+import org.jitsi.impl.neomedia.device.AudioSilenceMediaDevice;
 import org.jitsi.jigasi.*;
 import org.jitsi.jigasi.sounds.*;
 import org.jitsi.jigasi.transcription.*;
@@ -329,6 +330,7 @@ public void peerStateChanged(CallPeerChangeEvent evt)
             {
                 TranscribingAudioMixerMediaDevice mixer
                     = new TranscribingAudioMixerMediaDevice(
+                        new AudioSilenceMediaDevice(),
                        (receiveStream, buffer) ->
                             {
                                 receivedBuffer[0] = true;

src/main/java/org/jitsi/jigasi/transcription/AbstractTranscriptionService.java

@@ -0,0 +1,27 @@
+package org.jitsi.jigasi.transcription;
+
+import org.jitsi.impl.neomedia.device.AudioMixerMediaDevice;
+import org.jitsi.impl.neomedia.device.AudioSilenceMediaDevice;
+import org.jitsi.impl.neomedia.device.ReceiveStreamBufferListener;
+
+public abstract class AbstractTranscriptionService
+        implements TranscriptionService
+{
+    protected TranscribingAudioMixerMediaDevice mediaDevice = null;
+
+    /**
+     * Get the MediaDevice this transcriber is listening to for audio
+     *
+     * @return the AudioMixerMediaDevice which should receive all audio needed
+     * to be transcribed
+     */
+    public AudioMixerMediaDevice getMediaDevice(ReceiveStreamBufferListener listener)
+    {
+        if (this.mediaDevice == null)
+        {
+            this.mediaDevice = new TranscribingAudioMixerMediaDevice(new AudioSilenceMediaDevice(), listener);
+        }
+
+        return this.mediaDevice;
+    }
+}

src/main/java/org/jitsi/jigasi/transcription/GoogleCloudTranscriptionService.java

@@ -47,7 +47,7 @@
  * @author Damian Minkov
  */
 public class GoogleCloudTranscriptionService
-    implements TranscriptionService
+    extends AbstractTranscriptionService
 {
 
     /**

src/main/java/org/jitsi/jigasi/transcription/Participant.java

@@ -492,6 +492,10 @@ public void setChatMember(ChatRoomMember chatMember)
         this.chatMember = chatMember;
     }
 
+    public ChatRoomMember getChatMember() {
+        return this.chatMember;
+    }
+
     /**
      * Get the identifier in the JID of this participant
      *
@@ -520,6 +524,7 @@ void joined()
         {
             session = transcriber.getTranscriptionService()
                 .initStreamingSession(this);
+            logger.info("====Participant joined "+ this.getDebugName());
             session.addTranscriptionListener(this);
             sessions.put(getLanguageKey(), session);
             isCompleted = false;

src/main/java/org/jitsi/jigasi/transcription/Transcriber.java

@@ -29,6 +29,9 @@
 import org.jivesoftware.smack.packet.Presence;
 
 import javax.media.Buffer;
+import javax.media.Format;
+import javax.media.format.AudioFormat;
+import javax.media.protocol.CaptureDevice;
 import javax.media.rtp.*;
 import java.util.*;
 import java.util.concurrent.*;
@@ -142,11 +145,6 @@ private enum State
      */
     private Transcript transcript = new Transcript();
 
-    /**
-     * The MediaDevice which will get all audio to transcribe
-     */
-    private TranscribingAudioMixerMediaDevice mediaDevice
-        = new TranscribingAudioMixerMediaDevice(this);
 
     private static final String CUSTOM_TRANSLATION_SERVICE_PROP
             = "org.jitsi.jigasi.transcription.translationService";
@@ -174,7 +172,7 @@ private enum State
      * The service which is used to send audio and receive the
      * transcription of said audio
      */
-    private TranscriptionService transcriptionService;
+    private AbstractTranscriptionService transcriptionService;
 
     /**
      * A single thread which is used to manage the buffering and sending
@@ -214,7 +212,7 @@ private enum State
      */
     public Transcriber(String roomName,
                        String roomUrl,
-                       TranscriptionService service)
+                       AbstractTranscriptionService service)
     {
         if (!service.supportsStreamRecognition())
         {
@@ -242,7 +240,7 @@ public Transcriber(String roomName,
      * @param service the transcription service which will be used to transcribe
      * the audio streams
      */
-    public Transcriber(TranscriptionService service)
+    public Transcriber(AbstractTranscriptionService service)
     {
         this(null, null, service);
     }
@@ -832,7 +830,7 @@ public List<Participant> getParticipants()
      */
     public AudioMixerMediaDevice getMediaDevice()
     {
-        return this.mediaDevice;
+        return this.transcriptionService.getMediaDevice(this);
     }
 
     /**

src/main/java/org/jitsi/jigasi/transcription/TranscribingAudioMixerMediaDevice.java

@@ -18,6 +18,25 @@
 package org.jitsi.jigasi.transcription;
 
 import org.jitsi.impl.neomedia.device.*;
+import org.jitsi.impl.neomedia.format.MediaFormatImpl;
+import org.jitsi.impl.neomedia.jmfext.media.renderer.audio.AbstractAudioRenderer;
+import org.jitsi.jigasi.JigasiBundleActivator;
+import org.jitsi.service.neomedia.MediaDirection;
+import org.jitsi.service.neomedia.QualityPreset;
+import org.jitsi.service.neomedia.RTPExtension;
+import org.jitsi.service.neomedia.codec.EncodingConfiguration;
+import org.jitsi.service.neomedia.format.MediaFormat;
+import org.jitsi.utils.MediaType;
+import org.jitsi.utils.logging.Logger;
+
+import javax.media.Format;
+import javax.media.format.AudioFormat;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.security.Key;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 
 /**
  * AudioMixerMediaDevice which adds a {@link ReceiveStreamBufferListener} to
@@ -37,10 +56,14 @@ public class TranscribingAudioMixerMediaDevice
      * and has a listener for all other audio
      */
     public TranscribingAudioMixerMediaDevice(
-        ReceiveStreamBufferListener listener)
+            AudioSilenceMediaDevice device,
+            ReceiveStreamBufferListener listener)
     {
-        super(new AudioSilenceMediaDevice());
+        super(device);
+//        super(new WhisperTsAudioSilenceMediaDevice());
         super.setReceiveStreamBufferListener(listener);
     }
 
+
+
 }

src/main/java/org/jitsi/jigasi/transcription/VoskTranscriptionService.java

@@ -46,7 +46,7 @@
  * @author Nickolay V. Shmyrev
  */
 public class VoskTranscriptionService
-    implements TranscriptionService
+        extends AbstractTranscriptionService
 {
 
     /**

src/main/java/org/jitsi/jigasi/transcription/WhisperConnectionPool.java

@@ -0,0 +1,106 @@
+package org.jitsi.jigasi.transcription;
+
+import org.jitsi.utils.logging.Logger;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+
+public class WhisperConnectionPool
+{
+    /**
+     * The logger class
+     */
+    private final static Logger logger
+            = Logger.getLogger(WhisperConnectionPool.class);
+
+    /**
+     * The singleton instance to be returned
+     */
+    private static WhisperConnectionPool instance = null;
+
+    /**
+     * The participants which use the roomId connection
+     */
+    private Map<String, Set<String>> participants = new HashMap<>();
+
+    /**
+     * The connection pool
+     */
+    private HashMap<String, WhisperWebsocket> connections = new HashMap<>();
+
+    /**
+     * Gets a connection if it exists, creates one if it doesn't.
+     * @param roomId
+     * @param participantId
+     * @return
+     * @throws Exception
+     */
+    public WhisperWebsocket getConnection(String roomId, String participantId) throws Exception {
+        if (!this.connections.containsKey(roomId))
+        {
+            logger.debug("Room " + roomId + " doesn't exist. Creating a new connection.");
+            this.connections.put(roomId, new WhisperWebsocket());
+            HashSet participantSet = new HashSet();
+            participantSet.add(participantId);
+            this.participants.put(roomId, participantSet);
+        }
+        else
+        {
+            logger.debug("Participant " + participantId + " already exists in room " + roomId + ".");
+            this.participants.get(roomId).add(participantId);
+        }
+
+        return this.connections.get(roomId);
+    }
+
+    /**
+     * Ends the connection if all participants have left the room
+     * @param roomId
+     * @param participantId
+     * @throws IOException
+     */
+    public void end(String roomId, String participantId) throws IOException
+    {
+        Set<String> participantsSet = this.participants.get(roomId);
+        if (!participantsSet.contains(participantId))
+        {
+            return;
+        }
+
+        participantsSet.remove(participantId);
+        if (!participantsSet.isEmpty())
+        {
+            return;
+        }
+
+        WhisperWebsocket conn = this.connections.get(roomId);
+        conn.disconnectParticipant(participantId);
+        if (conn.ended) {
+            this.connections.remove(roomId);
+        }
+    }
+
+    /**
+     * Static method to return the instance of the class
+     * @return
+     * @throws Exception
+     */
+    public static WhisperConnectionPool getInstance() throws Exception
+    {
+        if (instance == null)
+        {
+            synchronized (WhisperConnectionPool.class)
+            {
+                if (instance == null)
+                {
+                    instance = new WhisperConnectionPool();
+                }
+            }
+        }
+        return instance;
+    }
+}