Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix sending local state to other participants #4558

Draft
wants to merge 27 commits into
base: master
Choose a base branch
from
Draft

Fix sending local state to other participants #4558

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
4165637
Remove Dagger related code from PeerConnectionWrapper
danxuliu Dec 5, 2024
acf0db1
Add dummy Log implementation to be used in tests
danxuliu Dec 5, 2024
bc29b0e
Add unit tests for receiving data channel messages
danxuliu Dec 5, 2024
1b4d80d
Unify log messages for received data channel messages
danxuliu Dec 5, 2024
f204662
Include data channel label in log message
danxuliu Dec 5, 2024
e9daa6e
Rename "sendChannelData" to "send"
danxuliu Dec 6, 2024
9384ef6
Send data channel messages using "status" data channel
danxuliu Dec 5, 2024
1ede068
Fix remote data channels not disposed when removing peer connection
danxuliu Dec 6, 2024
2f05103
Move variable declaration into try block
danxuliu Dec 6, 2024
0223d67
Rewrite method to return early
danxuliu Dec 6, 2024
3ebcd26
Split condition
danxuliu Dec 6, 2024
c77b385
Queue data channel messages sent when data channel is not open
danxuliu Dec 6, 2024
fbb7a71
Add logs for sending data channel messages
danxuliu Dec 6, 2024
b7a9bf7
Store data channel label
danxuliu Dec 9, 2024
f410b46
Fix "removePeerConnection" not being thread-safe
danxuliu Dec 8, 2024
b338693
Fix "send" not respecting order of pending messages
danxuliu Dec 9, 2024
a9c6947
Set "hasMCU" once its value is known
danxuliu Oct 24, 2024
c21596e
Add helper class to send messages to call participants
danxuliu Oct 25, 2024
8464040
Send data channel messages only to "video" peer connections
danxuliu Oct 25, 2024
79ed2e0
Add data model for local call participants
danxuliu Oct 25, 2024
e8705e6
Add helper class to broadcast the local participant state
danxuliu Nov 30, 2024
5fe9eec
Add support for sending data channel messages to a single participant
danxuliu Dec 5, 2024
c1303b1
Send current state to remote participants when they join
danxuliu Dec 11, 2024
1508acc
Move attributes to local variables
danxuliu Dec 19, 2024
811b7c0
Add support for sending signaling messages in the MessageSender
danxuliu Dec 19, 2024
7cfc8b9
Rename variable
danxuliu Dec 20, 2024
d094dc5
Send state also through signaling messages
danxuliu Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 83 additions & 58 deletions app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,13 @@ import com.nextcloud.talk.application.NextcloudTalkApplication.Companion.sharedA
import com.nextcloud.talk.call.CallParticipant
import com.nextcloud.talk.call.CallParticipantList
import com.nextcloud.talk.call.CallParticipantModel
import com.nextcloud.talk.call.LocalStateBroadcaster
import com.nextcloud.talk.call.LocalStateBroadcasterMcu
import com.nextcloud.talk.call.LocalStateBroadcasterNoMcu
import com.nextcloud.talk.call.MessageSender
import com.nextcloud.talk.call.MessageSenderMcu
import com.nextcloud.talk.call.MessageSenderNoMcu
import com.nextcloud.talk.call.MutableLocalCallParticipantModel
import com.nextcloud.talk.call.ReactionAnimator
import com.nextcloud.talk.chat.ChatActivity
import com.nextcloud.talk.data.user.model.User
Expand Down Expand Up @@ -242,6 +249,9 @@ class CallActivity : CallBaseActivity() {
private var signalingMessageReceiver: SignalingMessageReceiver? = null
private val internalSignalingMessageSender = InternalSignalingMessageSender()
private var signalingMessageSender: SignalingMessageSender? = null
private var messageSender: MessageSender? = null
private val localCallParticipantModel: MutableLocalCallParticipantModel = MutableLocalCallParticipantModel()
private var localStateBroadcaster: LocalStateBroadcaster? = null
private val offerAnswerNickProviders: MutableMap<String?, OfferAnswerNickProvider?> = HashMap()
private val callParticipantMessageListeners: MutableMap<String?, CallParticipantMessageListener> = HashMap()
private val selfPeerConnectionObserver: PeerConnectionObserver = CallActivitySelfPeerConnectionObserver()
Expand Down Expand Up @@ -1119,6 +1129,7 @@ class CallActivity : CallBaseActivity() {
localStream!!.addTrack(localVideoTrack)
localVideoTrack!!.setEnabled(false)
localVideoTrack!!.addSink(binding!!.selfVideoRenderer)
localCallParticipantModel.isVideoEnabled = false
}

private fun microphoneInitialization() {
Expand All @@ -1129,12 +1140,12 @@ class CallActivity : CallBaseActivity() {
localAudioTrack = peerConnectionFactory!!.createAudioTrack("NCa0", audioSource)
localAudioTrack!!.setEnabled(false)
localStream!!.addTrack(localAudioTrack)
localCallParticipantModel.isAudioEnabled = false
}

@SuppressLint("MissingPermission")
private fun startMicInputDetection() {
if (permissionUtil!!.isMicrophonePermissionGranted() && micInputAudioRecordThread == null) {
var isSpeakingLongTerm = false
micInputAudioRecorder = AudioRecord(
MediaRecorder.AudioSource.MIC,
SAMPLE_RATE,
Expand All @@ -1151,13 +1162,8 @@ class CallActivity : CallBaseActivity() {
micInputAudioRecorder.read(byteArr, 0, byteArr.size)
val isCurrentlySpeaking = abs(byteArr[0].toDouble()) > MICROPHONE_VALUE_THRESHOLD

if (microphoneOn && isCurrentlySpeaking && !isSpeakingLongTerm) {
isSpeakingLongTerm = true
sendIsSpeakingMessage(true)
} else if (!isCurrentlySpeaking && isSpeakingLongTerm) {
isSpeakingLongTerm = false
sendIsSpeakingMessage(false)
}
localCallParticipantModel.isSpeaking = isCurrentlySpeaking

Thread.sleep(MICROPHONE_VALUE_SLEEP)
}
}
Expand All @@ -1166,27 +1172,6 @@ class CallActivity : CallBaseActivity() {
}
}

@Suppress("Detekt.NestedBlockDepth")
private fun sendIsSpeakingMessage(isSpeaking: Boolean) {
val isSpeakingMessage: String =
if (isSpeaking) SIGNALING_MESSAGE_SPEAKING_STARTED else SIGNALING_MESSAGE_SPEAKING_STOPPED

if (isConnectionEstablished && othersInCall) {
if (!hasMCU) {
for (peerConnectionWrapper in peerConnectionWrapperList) {
peerConnectionWrapper.sendChannelData(DataChannelMessage(isSpeakingMessage))
}
} else {
for (peerConnectionWrapper in peerConnectionWrapperList) {
if (peerConnectionWrapper.sessionId == webSocketClient!!.sessionId) {
peerConnectionWrapper.sendChannelData(DataChannelMessage(isSpeakingMessage))
break
}
}
}
}
}

private fun createCameraCapturer(enumerator: CameraEnumerator?): VideoCapturer? {
val deviceNames = enumerator!!.deviceNames

Expand Down Expand Up @@ -1330,12 +1315,9 @@ class CallActivity : CallBaseActivity() {
}

private fun toggleMedia(enable: Boolean, video: Boolean) {
var message: String
if (video) {
message = SIGNALING_MESSAGE_VIDEO_OFF
if (enable) {
binding!!.cameraButton.alpha = OPACITY_ENABLED
message = SIGNALING_MESSAGE_VIDEO_ON
startVideoCapture()
} else {
binding!!.cameraButton.alpha = OPACITY_DISABLED
Expand All @@ -1349,36 +1331,22 @@ class CallActivity : CallBaseActivity() {
}
if (localStream != null && localStream!!.videoTracks.size > 0) {
localStream!!.videoTracks[0].setEnabled(enable)
localCallParticipantModel.isVideoEnabled = enable
}
if (enable) {
binding!!.selfVideoRenderer.visibility = View.VISIBLE
} else {
binding!!.selfVideoRenderer.visibility = View.INVISIBLE
}
} else {
message = SIGNALING_MESSAGE_AUDIO_OFF
if (enable) {
message = SIGNALING_MESSAGE_AUDIO_ON
binding!!.microphoneButton.alpha = OPACITY_ENABLED
} else {
binding!!.microphoneButton.alpha = OPACITY_DISABLED
}
if (localStream != null && localStream!!.audioTracks.size > 0) {
localStream!!.audioTracks[0].setEnabled(enable)
}
}
if (isConnectionEstablished) {
if (!hasMCU) {
for (peerConnectionWrapper in peerConnectionWrapperList) {
peerConnectionWrapper.sendChannelData(DataChannelMessage(message))
}
} else {
for (peerConnectionWrapper in peerConnectionWrapperList) {
if (peerConnectionWrapper.sessionId == webSocketClient!!.sessionId) {
peerConnectionWrapper.sendChannelData(DataChannelMessage(message))
break
}
}
localCallParticipantModel.isAudioEnabled = enable
}
}
}
Expand Down Expand Up @@ -1618,6 +1586,15 @@ class CallActivity : CallBaseActivity() {
signalingMessageReceiver!!.addListener(localParticipantMessageListener)
signalingMessageReceiver!!.addListener(offerMessageListener)
signalingMessageSender = internalSignalingMessageSender

hasMCU = false

messageSender = MessageSenderNoMcu(
signalingMessageSender,
callParticipants.keys,
peerConnectionWrapperList
)

joinRoomAndCall()
}
}
Expand Down Expand Up @@ -1755,6 +1732,15 @@ class CallActivity : CallBaseActivity() {
callParticipantList = CallParticipantList(signalingMessageReceiver)
callParticipantList!!.addObserver(callParticipantListObserver)

if (hasMCU) {
localStateBroadcaster = LocalStateBroadcasterMcu(localCallParticipantModel, messageSender)
} else {
localStateBroadcaster = LocalStateBroadcasterNoMcu(
localCallParticipantModel,
messageSender as MessageSenderNoMcu
)
}

val apiVersion = ApiUtils.getCallApiVersion(conversationUser, intArrayOf(ApiUtils.API_V4, 1))
ncApi!!.joinCall(
credentials,
Expand Down Expand Up @@ -1903,6 +1889,26 @@ class CallActivity : CallBaseActivity() {
signalingMessageReceiver!!.addListener(localParticipantMessageListener)
signalingMessageReceiver!!.addListener(offerMessageListener)
signalingMessageSender = webSocketClient!!.signalingMessageSender

// If the connection with the signaling server was not established yet the value will be false, but it will
// be overwritten with the right value once the response to the "hello" message is received.
hasMCU = webSocketClient!!.hasMCU()
Log.d(TAG, "hasMCU is $hasMCU")

if (hasMCU) {
messageSender = MessageSenderMcu(
signalingMessageSender,
callParticipants.keys,
peerConnectionWrapperList,
webSocketClient!!.sessionId
)
} else {
messageSender = MessageSenderNoMcu(
signalingMessageSender,
callParticipants.keys,
peerConnectionWrapperList
)
}
} else {
if (webSocketClient!!.isConnected && currentCallStatus === CallStatus.PUBLISHER_FAILED) {
webSocketClient!!.restartWebSocket()
Expand All @@ -1928,6 +1934,25 @@ class CallActivity : CallBaseActivity() {
when (webSocketCommunicationEvent.getType()) {
"hello" -> {
Log.d(TAG, "onMessageEvent 'hello'")

hasMCU = webSocketClient!!.hasMCU()
Log.d(TAG, "hasMCU is $hasMCU")

if (hasMCU) {
messageSender = MessageSenderMcu(
signalingMessageSender,
callParticipants.keys,
peerConnectionWrapperList,
webSocketClient!!.sessionId
)
} else {
messageSender = MessageSenderNoMcu(
signalingMessageSender,
callParticipants.keys,
peerConnectionWrapperList
)
}

if (!webSocketCommunicationEvent.getHashMap()!!.containsKey("oldResumeId")) {
if (currentCallStatus === CallStatus.RECONNECTING) {
hangup(false, false)
Expand Down Expand Up @@ -2076,6 +2101,9 @@ class CallActivity : CallBaseActivity() {
private fun hangupNetworkCalls(shutDownView: Boolean, endCallForAll: Boolean) {
Log.d(TAG, "hangupNetworkCalls. shutDownView=$shutDownView")
val apiVersion = ApiUtils.getCallApiVersion(conversationUser, intArrayOf(ApiUtils.API_V4, 1))
if (localStateBroadcaster != null) {
localStateBroadcaster!!.destroy()
}
if (callParticipantList != null) {
callParticipantList!!.removeObserver(callParticipantListObserver)
callParticipantList!!.destroy()
Expand Down Expand Up @@ -2136,8 +2164,6 @@ class CallActivity : CallBaseActivity() {
unchanged: Collection<Participant>
) {
Log.d(TAG, "handleCallParticipantsChanged")
hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient!!.hasMCU()
Log.d(TAG, " hasMCU is $hasMCU")

// The signaling session is the same as the Nextcloud session only when the MCU is not used.
var currentSessionId = callSession
Expand Down Expand Up @@ -2418,6 +2444,9 @@ class CallActivity : CallBaseActivity() {
callParticipantEventDisplayers[sessionId] = callParticipantEventDisplayer
callParticipantModel.addObserver(callParticipantEventDisplayer, callParticipantEventDisplayersHandler)
runOnUiThread { addParticipantDisplayItem(callParticipantModel, "video") }

localStateBroadcaster!!.handleCallParticipantAdded(callParticipant.callParticipantModel)

return callParticipant
}

Expand All @@ -2443,6 +2472,9 @@ class CallActivity : CallBaseActivity() {

private fun removeCallParticipant(sessionId: String?) {
val callParticipant = callParticipants.remove(sessionId) ?: return

localStateBroadcaster!!.handleCallParticipantRemoved(callParticipant.callParticipantModel)

val screenParticipantDisplayItemManager = screenParticipantDisplayItemManagers.remove(sessionId)
callParticipant.callParticipantModel.removeObserver(screenParticipantDisplayItemManager)
val callParticipantEventDisplayer = callParticipantEventDisplayers.remove(sessionId)
Expand Down Expand Up @@ -2563,7 +2595,7 @@ class CallActivity : CallBaseActivity() {
}

override fun onNext(aLong: Long) {
peerConnectionWrapper.sendChannelData(dataChannelMessage)
peerConnectionWrapper.send(dataChannelMessage)
}

override fun onError(e: Throwable) {
Expand Down Expand Up @@ -3260,12 +3292,5 @@ class CallActivity : CallBaseActivity() {
private const val Y_POS_NO_CALL_INFO: Float = 20f

private const val SESSION_ID_PREFFIX_END: Int = 4

private const val SIGNALING_MESSAGE_SPEAKING_STARTED = "speaking"
private const val SIGNALING_MESSAGE_SPEAKING_STOPPED = "stoppedSpeaking"
private const val SIGNALING_MESSAGE_VIDEO_ON = "videoOn"
private const val SIGNALING_MESSAGE_VIDEO_OFF = "videoOff"
private const val SIGNALING_MESSAGE_AUDIO_ON = "audioOn"
private const val SIGNALING_MESSAGE_AUDIO_OFF = "audioOff"
}
}
Loading
Loading