Change the logic for frame dispatch latency reduction

include/FrameDelayCalculator.h

@@ -11,7 +11,7 @@ class FrameDelayCalculator
 {
 public:
 
-	FrameDelayCalculator(int aUpdateRefsPacketEarlyThresholdMs, int aUpdateRefsPacketLateThresholdMs, std::chrono::milliseconds aUpdateRefsStepPacketEarlyMs);
+	FrameDelayCalculator(int aUpdateRefsPacketLateThresholdMs, std::chrono::milliseconds aUpdateRefsStepPacketEarlyMs);
 
 	/**
 	 * Calculate the dispatch delay for the arrived frame
@@ -40,15 +40,15 @@ class FrameDelayCalculator
 	// The following thresholds are checking the offsets of arrival time with regard to the previously
 	// scheduled time.
 
-	// Controls the stability of the dispatching delay.
-	int updateRefsPacketEarlyThresholdMs = 0;	
 	// Controls how accurate the a/v sync. Smaller means more in sync. Must >= 0. 
 	int updateRefsPacketLateThresholdMs = 0;
 	// Controls the latency reduction speed
 	std::chrono::milliseconds updateRefsStepPacketEarlyMs {0};
 
 	bool initialized = false;
-	bool reducingLatency = false;
+
+	// The start time when all streams arrive early. If packets become late, this time will be reset.
+	std::optional<std::chrono::milliseconds> allEarlyStartTimeMs;
 
 	std::chrono::milliseconds refTime {0};
 	uint64_t refTimestamp = 0;

include/FrameDispatchCoordinator.h

@@ -7,12 +7,10 @@
 class FrameDispatchCoordinator
 {
 public:	
-	static constexpr int DefaultUpdateRefsPacketEarlyThresholdMs = -200;
-	static constexpr int DefaultUpdateRefsPacketLateThresholdMs = 0;	
+	static constexpr int DefaultUpdateRefsPacketLateThresholdMs = 10;
 	static constexpr std::chrono::milliseconds DefaultUpdateRefsStepPacketEarlyMs = std::chrono::milliseconds(20);
 
-	FrameDispatchCoordinator(int aUpdateRefsPacketEarlyThresholdMs = DefaultUpdateRefsPacketEarlyThresholdMs,
-			int aUpdateRefsPacketLateThresholdMs = DefaultUpdateRefsPacketLateThresholdMs, 
+	FrameDispatchCoordinator(int aUpdateRefsPacketLateThresholdMs = DefaultUpdateRefsPacketLateThresholdMs, 
 			std::chrono::milliseconds aUpdateRefsStepPacketEarlyMs = DefaultUpdateRefsStepPacketEarlyMs);
 
 	/**

src/FrameDelayCalculator.cpp

@@ -22,10 +22,8 @@ static constexpr T convertTimestampClockRate(T ts, uint64_t originalRate, uint64
 static constexpr uint64_t UnifiedClockRate = 90 * 1000;
 }
 
-FrameDelayCalculator::FrameDelayCalculator(int aUpdateRefsPacketEarlyThresholdMs,
-					int aUpdateRefsPacketLateThresholdMs, 
+FrameDelayCalculator::FrameDelayCalculator(int aUpdateRefsPacketLateThresholdMs, 
 					std::chrono::milliseconds aUpdateRefsStepPacketEarlyMs) :
-	updateRefsPacketEarlyThresholdMs(aUpdateRefsPacketEarlyThresholdMs),
 	updateRefsPacketLateThresholdMs(aUpdateRefsPacketLateThresholdMs),
 	updateRefsStepPacketEarlyMs(aUpdateRefsStepPacketEarlyMs)
 {
@@ -53,20 +51,20 @@ std::chrono::milliseconds FrameDelayCalculator::OnFrame(uint64_t streamIdentifie
 
 	// We would delay the early arrived frame
 	std::chrono::milliseconds delayMs(-lateMs);
+	auto updateRefsPacketEarlyThresholdMs = -updateRefsStepPacketEarlyMs.count();
 
 	if (lateMs > updateRefsPacketLateThresholdMs)  // Packet late
 	{
 		refTime = now;
 		refTimestamp = unifiedTs;
 
-		// Stop reducing latency as a packet becomes late
-		reducingLatency = false;
+		allEarlyStartTimeMs.reset();
 		delayMs = std::chrono::milliseconds(0);
 	}
-	else if (!reducingLatency && lateMs < updateRefsPacketEarlyThresholdMs)  // Packet early
+	else if (lateMs < updateRefsPacketEarlyThresholdMs)  // Packet early
 	{
 		// Loop to see if all the streams have arrived earlier
-		reducingLatency = std::all_of(frameArrivalInfo.begin(), frameArrivalInfo.end(), 
+		bool allEarly = std::all_of(frameArrivalInfo.begin(), frameArrivalInfo.end(), 
 			[&](const auto& info) {
 				if (info.first == streamIdentifier) return true;
 
@@ -75,24 +73,36 @@ std::chrono::milliseconds FrameDelayCalculator::OnFrame(uint64_t streamIdentifie
 
 				return frameLateMs < updateRefsPacketEarlyThresholdMs;
 			});
+
+		if (allEarly)
+		{
+			if (!allEarlyStartTimeMs.has_value())
+				allEarlyStartTimeMs = now;
+		}
+		else
+		{
+			allEarlyStartTimeMs.reset();
+		}
 	}
-	else if (reducingLatency && lateMs > -updateRefsStepPacketEarlyMs.count())
+	else
 	{
-		// Stop reducing latency otherwise the frame would be regarded as late if we
-		// do a further step of latency reduction.
-		reducingLatency = false;
+		allEarlyStartTimeMs.reset();
 	}
 
 	frameArrivalInfo[streamIdentifier] = {now, unifiedTs};
-
-	if (reducingLatency)
+
+	// If all stream becomes ealier for a while (> 2s), we reduce the latency
+	if (allEarlyStartTimeMs.has_value() && (now - *allEarlyStartTimeMs) > 2000ms)
 	{
 		// Make reference time earlier for same time stamp, which means
 		// frames will be dispatched ealier.
 		refTime -= updateRefsStepPacketEarlyMs;
 
 		// Reduce the delay
 		delayMs -= updateRefsStepPacketEarlyMs;
+
+		// Restart the latency reduction process
+		allEarlyStartTimeMs.reset();
 	}
 
 	return std::max(delayMs, std::chrono::milliseconds(0));

src/FrameDispatchCoordinator.cpp

@@ -1,9 +1,8 @@
 #include "FrameDispatchCoordinator.h"
 
-FrameDispatchCoordinator::FrameDispatchCoordinator(int aUpdateRefsPacketEarlyThresholdMs,
-					int aUpdateRefsPacketLateThresholdMs, 
+FrameDispatchCoordinator::FrameDispatchCoordinator(int aUpdateRefsPacketLateThresholdMs, 
 					std::chrono::milliseconds aUpdateRefsStepPacketEarlyMs) :
-	frameDelayCalculator(aUpdateRefsPacketEarlyThresholdMs, aUpdateRefsPacketLateThresholdMs, aUpdateRefsStepPacketEarlyMs)
+	frameDelayCalculator(aUpdateRefsPacketLateThresholdMs, aUpdateRefsStepPacketEarlyMs)
 {
 }