From b2aeeb86ce019ddb3693d48acb11b61cf959a32d Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Mon, 16 Dec 2024 17:03:54 +0000 Subject: [PATCH] Add developer mode option to show RTC connection statistics --- locales/en/app.json | 1 + src/RTCConnectionStats.tsx | 73 +++++++++++++++ src/settings/DeveloperSettingsTab.tsx | 19 ++++ src/settings/settings.ts | 5 + src/state/MediaViewModel.ts | 127 +++++++++++++++++++++----- src/tile/GridTile.tsx | 4 + src/tile/MediaView.tsx | 11 +++ 7 files changed, 218 insertions(+), 22 deletions(-) create mode 100644 src/RTCConnectionStats.tsx diff --git a/locales/en/app.json b/locales/en/app.json index a47e5bebe..b42125f27 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -73,6 +73,7 @@ "duplicate_tiles_label": "Number of additional tile copies per participant", "hostname": "Hostname: {{hostname}}", "matrix_id": "Matrix ID: {{id}}", + "show_connection_stats": "Show connection statistics", "show_non_member_tiles": "Show tiles for non-member media" }, "disconnected_banner": "Connectivity to the server has been lost.", diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx new file mode 100644 index 000000000..c3c3b7a67 --- /dev/null +++ b/src/RTCConnectionStats.tsx @@ -0,0 +1,73 @@ +/* +Copyright 2023, 2024 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only +Please see LICENSE in the repository root for full details. +*/ + +import { type FC } from "react"; +import { Text, Tooltip, TooltipProvider } from "@vector-im/compound-web"; +import { + MicOnSolidIcon, + VideoCallSolidIcon, +} from "@vector-im/compound-design-tokens/assets/web/icons"; + +interface Props { + audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; +} + +export const RTCConnectionStats: FC = ({ audio, video, ...rest }) => { + return ( +
+ + {audio && ( +
+ + + + {"jitter" in audio && typeof audio.jitter === "number" && ( + +  {(audio.jitter * 1000).toFixed(0)}ms + + )} +
+ )} + {video && ( +
+ {video && ( + + + + )} + {video?.framesPerSecond && ( + +  {video.framesPerSecond.toFixed(0)}fps + + )} + {"jitter" in video && typeof video.jitter === "number" && ( + +  {(video.jitter * 1000).toFixed(0)}ms + + )} + {"frameHeight" in video && + typeof video.frameHeight === "number" && + "frameWidth" in video && + typeof video.frameWidth === "number" && ( + +  {video.frameWidth}x{video.frameHeight} + + )} + {"qualityLimitationReason" in video && + typeof video.qualityLimitationReason === "string" && + video.qualityLimitationReason !== "none" && ( + +  {video.qualityLimitationReason} + + )} +
+ )} + +
+ ); +}; diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx index 057b0b0c7..96ab262fc 100644 --- a/src/settings/DeveloperSettingsTab.tsx +++ b/src/settings/DeveloperSettingsTab.tsx @@ -14,6 +14,7 @@ import { duplicateTiles as duplicateTilesSetting, debugTileLayout as debugTileLayoutSetting, showNonMemberTiles as showNonMemberTilesSetting, + showConnectionStats as showConnectionStatsSetting, } from "./settings"; import type { MatrixClient } from "matrix-js-sdk/src/client"; @@ -31,6 +32,10 @@ export const DeveloperSettingsTab: FC = ({ client }) => { showNonMemberTilesSetting, ); + const [showConnectionStats, setShowConnectionStats] = useSetting( + showConnectionStatsSetting, + ); + return ( <>

@@ -103,6 +108,20 @@ export const DeveloperSettingsTab: FC = ({ client }) => { )} /> + + ): void => { + setShowConnectionStats(event.target.checked); + }, + [setShowConnectionStats], + )} + /> + ); }; diff --git a/src/settings/settings.ts b/src/settings/settings.ts index a902f9ab2..ae3c3a2e2 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -78,6 +78,11 @@ export const showNonMemberTiles = new Setting( ); export const debugTileLayout = new Setting("debug-tile-layout", false); +export const showConnectionStats = new Setting( + "show-connection-stats", + false, +); + export const audioInput = new Setting( "audio-input", undefined, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index ea015eb8b..99b48bb5c 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -47,7 +47,7 @@ import { useEffect } from "react"; import { ViewModel } from "./ViewModel"; import { useReactiveState } from "../useReactiveState"; -import { alwaysShowSelf } from "../settings/settings"; +import { alwaysShowSelf, showConnectionStats } from "../settings/settings"; import { accumulate } from "../utils/observable"; import { type EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; @@ -96,27 +96,23 @@ export function observeTrackReference( ); } -function observeRemoteTrackReceivingOkay( +export function observeRtpStreamStats( participant: Participant, source: Track.Source, -): Observable { - let lastStats: { - framesDecoded: number | undefined; - framesDropped: number | undefined; - framesReceived: number | undefined; - } = { - framesDecoded: undefined, - framesDropped: undefined, - framesReceived: undefined, - }; - + type: "inbound-rtp" | "outbound-rtp", +): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined +> { return combineLatest([ observeTrackReference(of(participant), source), interval(1000).pipe(startWith(0)), ]).pipe( switchMap(async ([trackReference]) => { const track = trackReference?.publication?.track; - if (!track || !(track instanceof RemoteTrack)) { + if ( + !track || + !(track instanceof RemoteTrack || track instanceof LocalTrack) + ) { return undefined; } const report = await track.getRTCStatsReport(); @@ -125,19 +121,59 @@ function observeRemoteTrackReceivingOkay( } for (const v of report.values()) { - if (v.type === "inbound-rtp") { - const { framesDecoded, framesDropped, framesReceived } = - v as RTCInboundRtpStreamStats; - return { - framesDecoded, - framesDropped, - framesReceived, - }; + if (v.type === type) { + return v; } } return undefined; }), + startWith(undefined), + ); +} + +export function observeInboundRtpStreamStats( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats(participant, source, "inbound-rtp").pipe( + map((x) => x as RTCInboundRtpStreamStats | undefined), + ); +} + +export function observeOutboundRtpStreamStats( + participant: Participant, + source: Track.Source, +): Observable { + return observeRtpStreamStats(participant, source, "outbound-rtp").pipe( + map((x) => x as RTCOutboundRtpStreamStats | undefined), + ); +} + +function observeRemoteTrackReceivingOkay( + participant: Participant, + source: Track.Source, +): Observable { + let lastStats: { + framesDecoded: number | undefined; + framesDropped: number | undefined; + framesReceived: number | undefined; + } = { + framesDecoded: undefined, + framesDropped: undefined, + framesReceived: undefined, + }; + + return observeInboundRtpStreamStats(participant, source).pipe( + map((stats) => { + if (!stats) return undefined; + const { framesDecoded, framesDropped, framesReceived } = stats; + return { + framesDecoded, + framesDropped, + framesReceived, + }; + }), filter((newStats) => !!newStats), map((newStats): boolean | undefined => { const oldStats = lastStats; @@ -401,6 +437,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { public get local(): boolean { return this instanceof LocalUserMediaViewModel; } + + public abstract get audioStreamStats(): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; + public abstract get videoStreamStats(): Observable< + RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined + >; } /** @@ -440,6 +483,26 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { ) { super(id, member, participant, encryptionSystem, livekitRoom); } + + public audioStreamStats = combineLatest([ + this.participant, + showConnectionStats.value, + ]).pipe( + switchMap(([p, enabled]) => { + if (!p || !enabled) return of(undefined); + return observeOutboundRtpStreamStats(p, Track.Source.Microphone); + }), + ); + + public videoStreamStats = combineLatest([ + this.participant, + showConnectionStats.value, + ]).pipe( + switchMap(([p, enabled]) => { + if (!p || !enabled) return of(undefined); + return observeOutboundRtpStreamStats(p, Track.Source.Camera); + }), + ); } /** @@ -519,6 +582,26 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { public commitLocalVolume(): void { this.localVolumeCommit.next(); } + + public audioStreamStats = combineLatest([ + this.participant, + showConnectionStats.value, + ]).pipe( + switchMap(([p, enabled]) => { + if (!p || !enabled) return of(undefined); + return observeInboundRtpStreamStats(p, Track.Source.Microphone); + }), + ); + + public videoStreamStats = combineLatest([ + this.participant, + showConnectionStats.value, + ]).pipe( + switchMap(([p, enabled]) => { + if (!p || !enabled) return of(undefined); + return observeInboundRtpStreamStats(p, Track.Source.Camera); + }), + ); } /** diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 73c175277..e5e73f9fb 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -86,6 +86,8 @@ const UserMediaTile = forwardRef( const video = useObservableEagerState(vm.video); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); const encryptionStatus = useObservableEagerState(vm.encryptionStatus); + const audioStreamStats = useObservableEagerState(vm.audioStreamStats); + const videoStreamStats = useObservableEagerState(vm.videoStreamStats); const audioEnabled = useObservableEagerState(vm.audioEnabled); const videoEnabled = useObservableEagerState(vm.videoEnabled); const speaking = useObservableEagerState(vm.speaking); @@ -176,6 +178,8 @@ const UserMediaTile = forwardRef( currentReaction={currentReaction} raisedHandOnClick={raisedHandOnClick} localParticipant={vm.local} + audioStreamStats={audioStreamStats} + videoStreamStats={videoStreamStats} {...props} /> ); diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 0d5341a82..abc3904b8 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -22,6 +22,7 @@ import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator"; import { showHandRaisedTimer, useSetting } from "../settings/settings"; import { type ReactionOption } from "../reactions"; import { ReactionIndicator } from "../reactions/ReactionIndicator"; +import { RTCConnectionStats } from "../RTCConnectionStats"; interface Props extends ComponentProps { className?: string; @@ -42,6 +43,8 @@ interface Props extends ComponentProps { currentReaction?: ReactionOption; raisedHandOnClick?: () => void; localParticipant: boolean; + audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; + videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats; } export const MediaView = forwardRef( @@ -65,6 +68,8 @@ export const MediaView = forwardRef( currentReaction, raisedHandOnClick, localParticipant, + audioStreamStats, + videoStreamStats, ...props }, ref, @@ -125,6 +130,12 @@ export const MediaView = forwardRef( {t("video_tile.waiting_for_media")} )} + {(audioStreamStats || videoStreamStats) && ( + + )} {/* TODO: Bring this back once encryption status is less broken */} {/*encryptionStatus !== EncryptionStatus.Okay && (