From 8b5bbd8b390cedc0ad6bf64857de778aae305489 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Tue, 13 Feb 2024 21:27:20 +1000 Subject: [PATCH 01/22] Change the label for the remote candidate --- .../library/src/PeerConnectionController/AggregatedStats.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 614301f7..32f2da35 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -171,7 +171,7 @@ export class AggregatedStats { */ handleRemoteCandidate(stat: CandidateStat) { const RemoteCandidate = new CandidateStat(); - RemoteCandidate.label = 'local-candidate'; + RemoteCandidate.label = 'remote-candidate'; RemoteCandidate.address = stat.address; RemoteCandidate.port = stat.port; RemoteCandidate.protocol = stat.protocol; From daff9981b93433644d9445de9f96c50c46d3eda9 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Tue, 13 Feb 2024 21:31:45 +1000 Subject: [PATCH 02/22] Added the following missing properties id,timestamp,type,lastPacketReceivedTimestamp,lastPacketSentTimestamp,priority,remoteCandidateId,transportId and writable --- .../PeerConnectionController/CandidatePairStats.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts b/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts index 6bb938ba..39d00a87 100644 --- a/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts +++ b/Frontend/library/src/PeerConnectionController/CandidatePairStats.ts @@ -6,12 +6,19 @@ export class CandidatePairStats { bytesReceived: number; bytesSent: number; + currentRoundTripTime: number; + id: string; + lastPacketReceivedTimestamp: number; + lastPacketSentTimestamp: number; localCandidateId: string; - remoteCandidateId: string; nominated: boolean; + priority: number; readable: boolean; - writable: boolean; + remoteCandidateId: string; selected: boolean; state: string; - currentRoundTripTime: number; + timestamp: number; + transportId: string; + type: string; + writable: boolean; } From 5542d91e137a2ffdfba29e319d89399994d99fc1 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Tue, 13 Feb 2024 21:32:53 +1000 Subject: [PATCH 03/22] Added relayProtocol and transport ID to the candidate Stat --- .../library/src/PeerConnectionController/CandidateStat.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/CandidateStat.ts b/Frontend/library/src/PeerConnectionController/CandidateStat.ts index d26fafae..03ceeee5 100644 --- a/Frontend/library/src/PeerConnectionController/CandidateStat.ts +++ b/Frontend/library/src/PeerConnectionController/CandidateStat.ts @@ -4,10 +4,12 @@ * ICE Candidate Stat collected from the RTC Stats Report */ export class CandidateStat { - label: string; - id: string; address: string; candidateType: string; + id: string; + label: string; port: number; protocol: 'tcp' | 'udp'; + relayProtocol: 'tcp' | 'udp' | 'tls'; + transportId: string; } From 1be9ed5d4d6c8d4dcb5701dc2954dc81668ccffe Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Tue, 13 Feb 2024 21:36:03 +1000 Subject: [PATCH 04/22] Added relayProtocol and transport ID to the candidate Stat --- .../library/src/PeerConnectionController/AggregatedStats.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 32f2da35..d9ad6305 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -162,6 +162,8 @@ export class AggregatedStats { localCandidate.protocol = stat.protocol; localCandidate.candidateType = stat.candidateType; localCandidate.id = stat.id; + localCandidate.relayProtocol = stat.relayProtocol; + localCandidate.transportId = stat.transportId; this.localCandidates.push(localCandidate); } @@ -177,6 +179,8 @@ export class AggregatedStats { RemoteCandidate.protocol = stat.protocol; RemoteCandidate.id = stat.id; RemoteCandidate.candidateType = stat.candidateType; + RemoteCandidate.relayProtocol = stat.relayProtocol; + RemoteCandidate.transportId = stat.transportId this.remoteCandidates.push(RemoteCandidate); } From 03ee5b6cdc2cd2936f189710351da6fcda182fbf Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Tue, 13 Feb 2024 21:37:53 +1000 Subject: [PATCH 05/22] Set the parsed candidate pair stat to the nominated and selected pair --- .../PeerConnectionController/AggregatedStats.ts | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index d9ad6305..18e69045 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -120,16 +120,11 @@ export class AggregatedStats { * @param stat - the stats coming in from ice candidates */ handleCandidatePair(stat: CandidatePairStats) { - this.candidatePair.bytesReceived = stat.bytesReceived; - this.candidatePair.bytesSent = stat.bytesSent; - this.candidatePair.localCandidateId = stat.localCandidateId; - this.candidatePair.remoteCandidateId = stat.remoteCandidateId; - this.candidatePair.nominated = stat.nominated; - this.candidatePair.readable = stat.readable; - this.candidatePair.selected = stat.selected; - this.candidatePair.writable = stat.writable; - this.candidatePair.state = stat.state; - this.candidatePair.currentRoundTripTime = stat.currentRoundTripTime; + + // If the candidate pair is nominated and selected set to the candidate pair + if (stat.nominated && stat.selected){ + this.candidatePair = stat; + } } /** From 7d97eb111151686b0c3e12092a4367d37f8c6ced Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Wed, 14 Feb 2024 13:24:37 +1000 Subject: [PATCH 06/22] Implemeneted a stream waring event --- Frontend/library/src/Util/EventEmitter.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index 64efd0ca..0314abcd 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -537,6 +537,23 @@ export class PlayerCountEvent extends Event { } } +/** + * An event that is emitted when the stream quality is degraded due to local network environment. + */ +export class StreamWarningEvent extends Event { + readonly type: 'streamWarning'; + readonly data: { + /** stream warning event */ + protocol: 'tcp' | 'udp' + relayProtocol: 'tcp' | 'udp' | 'tls' + candidateType: string + }; + constructor(data: StreamWarningEvent['data']) { + super('streamWarning'); + this.data = data; + } +} + export type PixelStreamingEvent = | AfkWarningActivateEvent | AfkWarningUpdateEvent @@ -557,6 +574,7 @@ export type PixelStreamingEvent = | StreamPreConnectEvent | StreamReconnectEvent | StreamPreDisconnectEvent + | StreamWarningEvent | PlayStreamErrorEvent | PlayStreamEvent | PlayStreamRejectedEvent From f05ce782a0e694d6c311e9f6e604ef316d6a6532 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Wed, 14 Feb 2024 13:25:36 +1000 Subject: [PATCH 07/22] Implemented the emitting of the stream warning event when if the local candidate is relayed over tcp --- .../src/PixelStreaming/PixelStreaming.ts | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index a8aca186..af739853 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -18,6 +18,7 @@ import { StreamPreConnectEvent, StreamReconnectEvent, StreamPreDisconnectEvent, + StreamWarningEvent, VideoEncoderAvgQPEvent, VideoInitializedEvent, WebRtcAutoConnectEvent, @@ -533,6 +534,30 @@ export class PixelStreaming { this._eventEmitter.dispatchEvent( new StatsReceivedEvent({ aggregatedStats: videoStats }) ); + + // Get the local candidate from the candidate pair + let localSelectedCandidate = videoStats.localCandidates.find((candid) => { + return candid.id == videoStats.candidatePair.localCandidateId + }) + + // Check if the local candidate is relaying over TCP + if (localSelectedCandidate.candidateType == 'relay' && localSelectedCandidate.relayProtocol == 'tcp'){ + + // Send a warning to the logger informing the user the stream will be severely degraded + Logger.Warning( + Logger.GetStackTrace(), + `Stream quality severely degraded, local connection is relayed over TCP due to the local network environment.` + ); + + // Emit a stream warning event + this._eventEmitter.dispatchEvent( + new StreamWarningEvent({ + candidateType: localSelectedCandidate.candidateType, + protocol: localSelectedCandidate.protocol, + relayProtocol: localSelectedCandidate.relayProtocol, + }) + ); + } } /** From 5b9f8b04e5ca7c5daf8e60effe69f672feb997a7 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Wed, 14 Feb 2024 13:25:54 +1000 Subject: [PATCH 08/22] Fixed up styling --- .../library/src/PeerConnectionController/AggregatedStats.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 18e69045..646f85bd 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -120,9 +120,8 @@ export class AggregatedStats { * @param stat - the stats coming in from ice candidates */ handleCandidatePair(stat: CandidatePairStats) { - - // If the candidate pair is nominated and selected set to the candidate pair - if (stat.nominated && stat.selected){ + // If the candidate pair has received bytes then set as the candidate pair + if (stat.bytesReceived > 0){ this.candidatePair = stat; } } From 1b8e9c349181ce4fdc5cd6b360b7a046069169f8 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 11:49:26 +1000 Subject: [PATCH 09/22] Refactored the candidate pair to an array, Implemneted a helper function to get the active candidate pair --- .../AggregatedStats.ts | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index 646f85bd..d9e0a7c6 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -25,7 +25,7 @@ export class AggregatedStats { inboundAudioStats: InboundAudioStats; lastVideoStats: InboundVideoStats; lastAudioStats: InboundAudioStats; - candidatePair: CandidatePairStats; + candidatePairs: Array; DataChannelStats: DataChannelStats; localCandidates: Array; remoteCandidates: Array; @@ -37,7 +37,6 @@ export class AggregatedStats { constructor() { this.inboundVideoStats = new InboundVideoStats(); this.inboundAudioStats = new InboundAudioStats(); - this.candidatePair = new CandidatePairStats(); this.DataChannelStats = new DataChannelStats(); this.outBoundVideoStats = new OutBoundVideoStats(); this.sessionStats = new SessionStats(); @@ -52,6 +51,7 @@ export class AggregatedStats { processStats(rtcStatsReport: RTCStatsReport) { this.localCandidates = new Array(); this.remoteCandidates = new Array(); + this.candidatePairs = new Array(); rtcStatsReport.forEach((stat) => { const type: RTCStatsTypePS = stat.type; @@ -120,10 +120,10 @@ export class AggregatedStats { * @param stat - the stats coming in from ice candidates */ handleCandidatePair(stat: CandidatePairStats) { - // If the candidate pair has received bytes then set as the candidate pair - if (stat.bytesReceived > 0){ - this.candidatePair = stat; - } + + // Add the candidate pair to the candidate pair array + this.candidatePairs.push(stat) + } /** @@ -306,4 +306,17 @@ export class AggregatedStats { isNumber(value: unknown): boolean { return typeof value === 'number' && isFinite(value); } + + /** + * Helper function to return the active candidate pair + * @returns The candidate pair that is currently receiving data + */ + public getActiveCandidatePair(): CandidatePairStats | null { + this.candidatePairs.forEach((candidatePair) => { + if (candidatePair.bytesReceived >0) { + return candidatePair + } + }) + return null + } } From 86fbdb26a82fb648587f208086d9a8b8304f2a81 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 12:54:32 +1000 Subject: [PATCH 10/22] Refactored to use Array.find --- .../src/PeerConnectionController/AggregatedStats.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts index d9e0a7c6..8a701194 100644 --- a/Frontend/library/src/PeerConnectionController/AggregatedStats.ts +++ b/Frontend/library/src/PeerConnectionController/AggregatedStats.ts @@ -312,11 +312,6 @@ export class AggregatedStats { * @returns The candidate pair that is currently receiving data */ public getActiveCandidatePair(): CandidatePairStats | null { - this.candidatePairs.forEach((candidatePair) => { - if (candidatePair.bytesReceived >0) { - return candidatePair - } - }) - return null + return this.candidatePairs.find((candidatePair) => candidatePair.bytesReceived > 0, null) } } From 0e89638f3ca16e450af44926e5869d065525abcd Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 12:56:24 +1000 Subject: [PATCH 11/22] Replaced the stream warning event with WebRtcTCPRelayDetectedEvent --- Frontend/library/src/Util/EventEmitter.ts | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/Frontend/library/src/Util/EventEmitter.ts b/Frontend/library/src/Util/EventEmitter.ts index 0314abcd..de1d433d 100644 --- a/Frontend/library/src/Util/EventEmitter.ts +++ b/Frontend/library/src/Util/EventEmitter.ts @@ -538,19 +538,12 @@ export class PlayerCountEvent extends Event { } /** - * An event that is emitted when the stream quality is degraded due to local network environment. + * An event that is emitted when the webRTC connections is relayed over TCP. */ -export class StreamWarningEvent extends Event { - readonly type: 'streamWarning'; - readonly data: { - /** stream warning event */ - protocol: 'tcp' | 'udp' - relayProtocol: 'tcp' | 'udp' | 'tls' - candidateType: string - }; - constructor(data: StreamWarningEvent['data']) { - super('streamWarning'); - this.data = data; +export class WebRtcTCPRelayDetectedEvent extends Event { + readonly type: 'webRtcTCPRelayDetected'; + constructor() { + super('webRtcTCPRelayDetected'); } } @@ -574,7 +567,6 @@ export type PixelStreamingEvent = | StreamPreConnectEvent | StreamReconnectEvent | StreamPreDisconnectEvent - | StreamWarningEvent | PlayStreamErrorEvent | PlayStreamEvent | PlayStreamRejectedEvent @@ -591,7 +583,8 @@ export type PixelStreamingEvent = | XrSessionStartedEvent | XrSessionEndedEvent | XrFrameEvent - | PlayerCountEvent; + | PlayerCountEvent + | WebRtcTCPRelayDetectedEvent; export class EventEmitter extends EventTarget { /** From bdf41af1731517df660c7c9e4544b0a44bcbae88 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 12:57:35 +1000 Subject: [PATCH 12/22] Implemeneted the WebRtcTCPRelayDetectedEvent to dispatch if the web rtc transport is tcp and the candidate type is relay --- .../src/PixelStreaming/PixelStreaming.ts | 48 ++++++++++--------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index af739853..a1826858 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -18,7 +18,6 @@ import { StreamPreConnectEvent, StreamReconnectEvent, StreamPreDisconnectEvent, - StreamWarningEvent, VideoEncoderAvgQPEvent, VideoInitializedEvent, WebRtcAutoConnectEvent, @@ -29,7 +28,8 @@ import { WebRtcSdpEvent, DataChannelLatencyTestResponseEvent, DataChannelLatencyTestResultEvent, - PlayerCountEvent + PlayerCountEvent, + WebRtcTCPRelayDetectedEvent } from '../Util/EventEmitter'; import { MessageOnScreenKeyboard } from '../WebSockets/MessageReceive'; import { WebXRController } from '../WebXR/WebXRController'; @@ -80,6 +80,9 @@ export class PixelStreaming { private _eventEmitter: EventEmitter; + private _webRtcTCPRelayDetectedCheck: boolean; + + /** * @param config - A newly instantiated config object * @param overrides - Parameters to override default behaviour @@ -87,6 +90,7 @@ export class PixelStreaming { */ constructor(config: Config, overrides?: PixelStreamingOverrides) { this.config = config; + this._webRtcTCPRelayDetectedCheck = false; if (overrides?.videoElementParent) { this._videoElementParent = overrides.videoElementParent; @@ -535,28 +539,28 @@ export class PixelStreaming { new StatsReceivedEvent({ aggregatedStats: videoStats }) ); - // Get the local candidate from the candidate pair - let localSelectedCandidate = videoStats.localCandidates.find((candid) => { - return candid.id == videoStats.candidatePair.localCandidateId - }) - - // Check if the local candidate is relaying over TCP - if (localSelectedCandidate.candidateType == 'relay' && localSelectedCandidate.relayProtocol == 'tcp'){ + // Check if the web rtc rely detected check has completed + if (!this._webRtcTCPRelayDetectedCheck){ - // Send a warning to the logger informing the user the stream will be severely degraded - Logger.Warning( - Logger.GetStackTrace(), - `Stream quality severely degraded, local connection is relayed over TCP due to the local network environment.` - ); + // Get the active candidate pair + let activeCandidatePair = videoStats.getActiveCandidatePair(); - // Emit a stream warning event - this._eventEmitter.dispatchEvent( - new StreamWarningEvent({ - candidateType: localSelectedCandidate.candidateType, - protocol: localSelectedCandidate.protocol, - relayProtocol: localSelectedCandidate.relayProtocol, - }) - ); + // Check if the active candidate pair is not null + if (activeCandidatePair != null){ + + // Get the local candidate assigned to the active candidate pair + let localCandidate = videoStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId,null) + + // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp + if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp'){ + + // Send the web rtc tcp relay detected Evebt + this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); + } + + // set the web rtc tcp relay detected check to true + this._webRtcTCPRelayDetectedCheck = true; + } } } From 55d02d69219e7bd521a8c977e2794bedb716b591 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 13:31:08 +1000 Subject: [PATCH 13/22] Refactored the stats panel to use the getActiveCandidatePair --- Frontend/ui-library/src/UI/StatsPanel.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index ee364c4e..a6043227 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -321,11 +321,11 @@ export class StatsPanel { // RTT const netRTT = Object.prototype.hasOwnProperty.call( - stats.candidatePair, + stats.getActiveCandidatePair(), 'currentRoundTripTime' - ) && stats.isNumber(stats.candidatePair.currentRoundTripTime) + ) && stats.isNumber(stats.getActiveCandidatePair().currentRoundTripTime) ? numberFormat.format( - stats.candidatePair.currentRoundTripTime * 1000 + stats.getActiveCandidatePair().currentRoundTripTime * 1000 ) : "Can't calculate"; this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT); From fb6bf16f77a074732cae2ffac95522f0292a2af9 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 17:27:23 +1000 Subject: [PATCH 14/22] Refactored the webrtc tcp detect check to act on an event instead of relying on a check --- .../src/PixelStreaming/PixelStreaming.ts | 60 ++++++++++--------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index a1826858..ffc6ba57 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -80,9 +80,6 @@ export class PixelStreaming { private _eventEmitter: EventEmitter; - private _webRtcTCPRelayDetectedCheck: boolean; - - /** * @param config - A newly instantiated config object * @param overrides - Parameters to override default behaviour @@ -90,7 +87,6 @@ export class PixelStreaming { */ constructor(config: Config, overrides?: PixelStreamingOverrides) { this.config = config; - this._webRtcTCPRelayDetectedCheck = false; if (overrides?.videoElementParent) { this._videoElementParent = overrides.videoElementParent; @@ -121,8 +117,40 @@ export class PixelStreaming { this.onScreenKeyboardHelper.showOnScreenKeyboard(command); this._webXrController = new WebXRController(this._webRtcController); + + // Add event listener for the webRtcConnected event + this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => { + + // Create a function to handle the web rtc tcp detect check the stats received event + let webRTCTCPdetectCheck = (statsReceivedEvent: StatsReceivedEvent) => { + + // Get the active candidate pair + let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair(); + + // Check if the active candidate pair is not null + if (activeCandidatePair != null) { + + // Get the local candidate assigned to the active candidate pair + let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null) + + // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp + if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') { + + // Send the web rtc tcp relay detected Evebt + this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); + } + // The check is completed and the stats listen event can be removed + this._eventEmitter.removeEventListener("statsReceived", webRTCTCPdetectCheck); + } + } + + // Bind to the stats received event + this._eventEmitter.addEventListener("statsReceived", webRTCTCPdetectCheck); + }); } + + /** * Gets the element that contains the video stream element. */ @@ -538,30 +566,6 @@ export class PixelStreaming { this._eventEmitter.dispatchEvent( new StatsReceivedEvent({ aggregatedStats: videoStats }) ); - - // Check if the web rtc rely detected check has completed - if (!this._webRtcTCPRelayDetectedCheck){ - - // Get the active candidate pair - let activeCandidatePair = videoStats.getActiveCandidatePair(); - - // Check if the active candidate pair is not null - if (activeCandidatePair != null){ - - // Get the local candidate assigned to the active candidate pair - let localCandidate = videoStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId,null) - - // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp - if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp'){ - - // Send the web rtc tcp relay detected Evebt - this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); - } - - // set the web rtc tcp relay detected check to true - this._webRtcTCPRelayDetectedCheck = true; - } - } } /** From 533d88a18a2ed41ab43a5bb15bc04707d5830b80 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 17:34:03 +1000 Subject: [PATCH 15/22] Refactored the getting of the active candidate pair to be stored as a variable instead of calling multiple times --- Frontend/ui-library/src/UI/StatsPanel.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index a6043227..d0c1922a 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -318,14 +318,17 @@ export class StatsPanel { ); } + // Store the active candidate pair + let activeCandidatePair = stats.getActiveCandidatePair(); + // RTT const netRTT = Object.prototype.hasOwnProperty.call( - stats.getActiveCandidatePair(), + activeCandidatePair, 'currentRoundTripTime' - ) && stats.isNumber(stats.getActiveCandidatePair().currentRoundTripTime) + ) && stats.isNumber(activeCandidatePair.currentRoundTripTime) ? numberFormat.format( - stats.getActiveCandidatePair().currentRoundTripTime * 1000 + activeCandidatePair.currentRoundTripTime * 1000 ) : "Can't calculate"; this.addOrUpdateStat('RTTStat', 'Net RTT (ms)', netRTT); From 9d262397f844150f44c579f2d15f04aac319e947 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 17:41:22 +1000 Subject: [PATCH 16/22] Updated the candidate pair to the candidate pairs array --- Frontend/library/src/PixelStreaming/PixelStreaming.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts index 71d86c21..0df5648b 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.test.ts @@ -398,9 +398,9 @@ describe('PixelStreaming', () => { expect.objectContaining({ data: { aggregatedStats: expect.objectContaining({ - candidatePair: expect.objectContaining({ - bytesReceived: 123 - }), + candidatePairs: [ + expect.objectContaining({ bytesReceived: 123 }) + ], localCandidates: [ expect.objectContaining({ address: 'mock-address' }) ] From 7eab39db15f676ab2b01d4072bfd4fec2aff0040 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Thu, 15 Feb 2024 18:04:13 +1000 Subject: [PATCH 17/22] Added a warning if the stream is relayed over tcp --- Frontend/ui-library/src/Application/Application.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Frontend/ui-library/src/Application/Application.ts b/Frontend/ui-library/src/Application/Application.ts index e17ce403..40739e48 100644 --- a/Frontend/ui-library/src/Application/Application.ts +++ b/Frontend/ui-library/src/Application/Application.ts @@ -378,6 +378,14 @@ export class Application { ({ data: { count }}) => this.onPlayerCount(count) ); + this.stream.addEventListener( + 'webRtcTCPRelayDetected', + ({}) => + Logger.Warning( + Logger.GetStackTrace(), + `Stream quailty degraded due to network enviroment, stream is relayed over TCP.` + ) + ); } /** From 51babfdfb6be703c76244556546e9698471612a7 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Fri, 16 Feb 2024 09:46:01 +1000 Subject: [PATCH 18/22] Moved the logic for emitting the webrtc tcp relay detect event to it's own function --- .../src/PixelStreaming/PixelStreaming.ts | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index ffc6ba57..f579cfe8 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -63,6 +63,7 @@ export class PixelStreaming { protected _webRtcController: WebRtcPlayerController; protected _webXrController: WebXRController; protected _dataChannelLatencyTestController: DataChannelLatencyTestController; + /** * Configuration object. You can read or modify config through this object. Whenever * the configuration is changed, the library will emit a `settingsChanged` event. @@ -118,39 +119,17 @@ export class PixelStreaming { this._webXrController = new WebXRController(this._webRtcController); + this._setupWebRtcTCPRelayDetection = this._setupWebRtcTCPRelayDetection.bind(this) + // Add event listener for the webRtcConnected event this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => { - - // Create a function to handle the web rtc tcp detect check the stats received event - let webRTCTCPdetectCheck = (statsReceivedEvent: StatsReceivedEvent) => { - - // Get the active candidate pair - let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair(); - - // Check if the active candidate pair is not null - if (activeCandidatePair != null) { - - // Get the local candidate assigned to the active candidate pair - let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null) - - // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp - if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') { - - // Send the web rtc tcp relay detected Evebt - this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); - } - // The check is completed and the stats listen event can be removed - this._eventEmitter.removeEventListener("statsReceived", webRTCTCPdetectCheck); - } - } // Bind to the stats received event - this._eventEmitter.addEventListener("statsReceived", webRTCTCPdetectCheck); + this._eventEmitter.addEventListener("statsReceived", this._setupWebRtcTCPRelayDetection); + }); } - - /** * Gets the element that contains the video stream element. */ @@ -660,6 +639,28 @@ export class PixelStreaming { ); } + // Sets up to emit the webrtc tcp relay detect event + _setupWebRtcTCPRelayDetection(statsReceivedEvent: StatsReceivedEvent) { + // Get the active candidate pair + let activeCandidatePair = statsReceivedEvent.data.aggregatedStats.getActiveCandidatePair(); + + // Check if the active candidate pair is not null + if (activeCandidatePair != null) { + + // Get the local candidate assigned to the active candidate pair + let localCandidate = statsReceivedEvent.data.aggregatedStats.localCandidates.find((candidate) => candidate.id == activeCandidatePair.localCandidateId, null) + + // Check if the local candidate is not null, candidate type is relay and the relay protocol is tcp + if (localCandidate != null && localCandidate.candidateType == 'relay' && localCandidate.relayProtocol == 'tcp') { + + // Send the web rtc tcp relay detected event + this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); + } + // The check is completed and the stats listen event can be removed + this._eventEmitter.removeEventListener("statsReceived", this._setupWebRtcTCPRelayDetection); + } + } + /** * Request a connection latency test. * NOTE: There are plans to refactor all request* functions. Expect changes if you use this! From d00bf7599ef91cdb3eb0bee0feb9d6bdbb7f602c Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Fri, 16 Feb 2024 12:53:55 +1000 Subject: [PATCH 19/22] Refactored the binding of the stats recieved event to the setup web rtc relay detected function --- Frontend/library/src/PixelStreaming/PixelStreaming.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index f579cfe8..475fa9c7 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -119,17 +119,18 @@ export class PixelStreaming { this._webXrController = new WebXRController(this._webRtcController); - this._setupWebRtcTCPRelayDetection = this._setupWebRtcTCPRelayDetection.bind(this) - // Add event listener for the webRtcConnected event this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => { // Bind to the stats received event - this._eventEmitter.addEventListener("statsReceived", this._setupWebRtcTCPRelayDetection); - + this._eventEmitter.addEventListener("statsReceived", (statsReceivedEvent: StatsReceivedEvent) => { this._setupWebRtcTCPRelayDetection(statsReceivedEvent)}); }); } + + + + /** * Gets the element that contains the video stream element. */ @@ -657,7 +658,7 @@ export class PixelStreaming { this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); } // The check is completed and the stats listen event can be removed - this._eventEmitter.removeEventListener("statsReceived", this._setupWebRtcTCPRelayDetection); + this._eventEmitter.removeEventListener("statsReceived", (statsReceivedEvent: StatsReceivedEvent) => { this._setupWebRtcTCPRelayDetection(statsReceivedEvent)}); } } From 8567df610d8b51d559fcdb3905527e29f5f20de0 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Fri, 16 Feb 2024 13:32:03 +1000 Subject: [PATCH 20/22] Removed un needed new lines --- Frontend/library/src/PixelStreaming/PixelStreaming.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index 475fa9c7..21b32915 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -127,10 +127,6 @@ export class PixelStreaming { }); } - - - - /** * Gets the element that contains the video stream element. */ From 15ed4ad2cb8d0a4ce1d5775fd87a186189de3290 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Fri, 16 Feb 2024 13:33:21 +1000 Subject: [PATCH 21/22] Added a check if the get active candidate is null to return a new candidate pair --- Frontend/ui-library/src/UI/StatsPanel.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Frontend/ui-library/src/UI/StatsPanel.ts b/Frontend/ui-library/src/UI/StatsPanel.ts index d0c1922a..e5436639 100644 --- a/Frontend/ui-library/src/UI/StatsPanel.ts +++ b/Frontend/ui-library/src/UI/StatsPanel.ts @@ -1,7 +1,7 @@ // Copyright Epic Games, Inc. All Rights Reserved. import { LatencyTest } from './LatencyTest'; -import { InitialSettings, Logger, PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4'; +import { CandidatePairStats, InitialSettings, Logger, PixelStreaming } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4'; import { AggregatedStats } from '@epicgames-ps/lib-pixelstreamingfrontend-ue5.4'; import { MathUtils } from '../Util/MathUtils'; import {DataChannelLatencyTest} from "./DataChannelLatencyTest"; @@ -318,8 +318,8 @@ export class StatsPanel { ); } - // Store the active candidate pair - let activeCandidatePair = stats.getActiveCandidatePair(); + // Store the active candidate pair return a new Candidate pair stat if getActiveCandidate is null + let activeCandidatePair = stats.getActiveCandidatePair() != null ? stats.getActiveCandidatePair() : new CandidatePairStats(); // RTT const netRTT = From ad24cc9b9b850b0d55aaeb47487528087e475820 Mon Sep 17 00:00:00 2001 From: David MacPherson Date: Mon, 19 Feb 2024 10:24:09 +1000 Subject: [PATCH 22/22] Reverted the refactor the binding of the stats recieved event function --- Frontend/library/src/PixelStreaming/PixelStreaming.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Frontend/library/src/PixelStreaming/PixelStreaming.ts b/Frontend/library/src/PixelStreaming/PixelStreaming.ts index 21b32915..1b18e7c7 100644 --- a/Frontend/library/src/PixelStreaming/PixelStreaming.ts +++ b/Frontend/library/src/PixelStreaming/PixelStreaming.ts @@ -123,7 +123,7 @@ export class PixelStreaming { this._eventEmitter.addEventListener("webRtcConnected", (webRtcConnectedEvent: WebRtcConnectedEvent) => { // Bind to the stats received event - this._eventEmitter.addEventListener("statsReceived", (statsReceivedEvent: StatsReceivedEvent) => { this._setupWebRtcTCPRelayDetection(statsReceivedEvent)}); + this._eventEmitter.addEventListener("statsReceived", this._setupWebRtcTCPRelayDetection.bind(this)); }); } @@ -654,7 +654,7 @@ export class PixelStreaming { this._eventEmitter.dispatchEvent(new WebRtcTCPRelayDetectedEvent()); } // The check is completed and the stats listen event can be removed - this._eventEmitter.removeEventListener("statsReceived", (statsReceivedEvent: StatsReceivedEvent) => { this._setupWebRtcTCPRelayDetection(statsReceivedEvent)}); + this._eventEmitter.removeEventListener("statsReceived", this._setupWebRtcTCPRelayDetection); } }