Add developer mode option to show RTC connection statistics

element-hq · Dec 16, 2024 · b2aeeb8 · b2aeeb8
1 parent e4bd9d7
commit b2aeeb8
Show file tree

Hide file tree

Showing 7 changed files with 218 additions and 22 deletions.
diff --git a/locales/en/app.json b/locales/en/app.json
@@ -73,6 +73,7 @@
     "duplicate_tiles_label": "Number of additional tile copies per participant",
     "hostname": "Hostname: {{hostname}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "show_connection_stats": "Show connection statistics",
     "show_non_member_tiles": "Show tiles for non-member media"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",

diff --git a/src/RTCConnectionStats.tsx b/src/RTCConnectionStats.tsx
@@ -0,0 +1,73 @@
+/*
+Copyright 2023, 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type FC } from "react";
+import { Text, Tooltip, TooltipProvider } from "@vector-im/compound-web";
+import {
+  MicOnSolidIcon,
+  VideoCallSolidIcon,
+} from "@vector-im/compound-design-tokens/assets/web/icons";
+
+interface Props {
+  audio?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+  video?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+}
+
+export const RTCConnectionStats: FC<Props> = ({ audio, video, ...rest }) => {
+  return (
+    <div>
+      <TooltipProvider>
+        {audio && (
+          <div>
+            <Tooltip label={JSON.stringify(audio, null, 2)}>
+              <MicOnSolidIcon />
+            </Tooltip>
+            {"jitter" in audio && typeof audio.jitter === "number" && (
+              <Text as="span" size="xs">
+                &nbsp;{(audio.jitter * 1000).toFixed(0)}ms
+              </Text>
+            )}
+          </div>
+        )}
+        {video && (
+          <div>
+            {video && (
+              <Tooltip label={JSON.stringify(video, null, 2)}>
+                <VideoCallSolidIcon />
+              </Tooltip>
+            )}
+            {video?.framesPerSecond && (
+              <Text as="span" size="xs">
+                &nbsp;{video.framesPerSecond.toFixed(0)}fps
+              </Text>
+            )}
+            {"jitter" in video && typeof video.jitter === "number" && (
+              <Text as="span" size="xs">
+                &nbsp;{(video.jitter * 1000).toFixed(0)}ms
+              </Text>
+            )}
+            {"frameHeight" in video &&
+              typeof video.frameHeight === "number" &&
+              "frameWidth" in video &&
+              typeof video.frameWidth === "number" && (
+                <Text as="span" size="xs">
+                  &nbsp;{video.frameWidth}x{video.frameHeight}
+                </Text>
+              )}
+            {"qualityLimitationReason" in video &&
+              typeof video.qualityLimitationReason === "string" &&
+              video.qualityLimitationReason !== "none" && (
+                <Text as="span" size="xs">
+                  &nbsp;{video.qualityLimitationReason}
+                </Text>
+              )}
+          </div>
+        )}
+      </TooltipProvider>
+    </div>
+  );
+};
diff --git a/src/settings/DeveloperSettingsTab.tsx b/src/settings/DeveloperSettingsTab.tsx
@@ -14,6 +14,7 @@ import {
   duplicateTiles as duplicateTilesSetting,
   debugTileLayout as debugTileLayoutSetting,
   showNonMemberTiles as showNonMemberTilesSetting,
+  showConnectionStats as showConnectionStatsSetting,
 } from "./settings";
 import type { MatrixClient } from "matrix-js-sdk/src/client";
 
@@ -31,6 +32,10 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
     showNonMemberTilesSetting,
   );
 
+  const [showConnectionStats, setShowConnectionStats] = useSetting(
+    showConnectionStatsSetting,
+  );
+
   return (
     <>
       <p>
@@ -103,6 +108,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client }) => {
           )}
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="showConnectionStats"
+          type="checkbox"
+          label={t("developer_mode.show_connection_stats")}
+          checked={!!showConnectionStats}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setShowConnectionStats(event.target.checked);
+            },
+            [setShowConnectionStats],
+          )}
+        />
+      </FieldRow>
     </>
   );
 };
diff --git a/src/settings/settings.ts b/src/settings/settings.ts
@@ -78,6 +78,11 @@ export const showNonMemberTiles = new Setting<boolean>(
 );
 export const debugTileLayout = new Setting("debug-tile-layout", false);
 
+export const showConnectionStats = new Setting<boolean>(
+  "show-connection-stats",
+  false,
+);
+
 export const audioInput = new Setting<string | undefined>(
   "audio-input",
   undefined,

diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts
@@ -47,7 +47,7 @@ import { useEffect } from "react";
 
 import { ViewModel } from "./ViewModel";
 import { useReactiveState } from "../useReactiveState";
-import { alwaysShowSelf } from "../settings/settings";
+import { alwaysShowSelf, showConnectionStats } from "../settings/settings";
 import { accumulate } from "../utils/observable";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
@@ -96,27 +96,23 @@ export function observeTrackReference(
   );
 }
 
-function observeRemoteTrackReceivingOkay(
+export function observeRtpStreamStats(
   participant: Participant,
   source: Track.Source,
-): Observable<boolean | undefined> {
-  let lastStats: {
-    framesDecoded: number | undefined;
-    framesDropped: number | undefined;
-    framesReceived: number | undefined;
-  } = {
-    framesDecoded: undefined,
-    framesDropped: undefined,
-    framesReceived: undefined,
-  };
-
+  type: "inbound-rtp" | "outbound-rtp",
+): Observable<
+  RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+> {
   return combineLatest([
     observeTrackReference(of(participant), source),
     interval(1000).pipe(startWith(0)),
   ]).pipe(
     switchMap(async ([trackReference]) => {
       const track = trackReference?.publication?.track;
-      if (!track || !(track instanceof RemoteTrack)) {
+      if (
+        !track ||
+        !(track instanceof RemoteTrack || track instanceof LocalTrack)
+      ) {
         return undefined;
       }
       const report = await track.getRTCStatsReport();
@@ -125,19 +121,59 @@ function observeRemoteTrackReceivingOkay(
       }
 
       for (const v of report.values()) {
-        if (v.type === "inbound-rtp") {
-          const { framesDecoded, framesDropped, framesReceived } =
-            v as RTCInboundRtpStreamStats;
-          return {
-            framesDecoded,
-            framesDropped,
-            framesReceived,
-          };
+        if (v.type === type) {
+          return v;
         }
       }
 
       return undefined;
     }),
+    startWith(undefined),
+  );
+}
+
+export function observeInboundRtpStreamStats(
+  participant: Participant,
+  source: Track.Source,
+): Observable<RTCInboundRtpStreamStats | undefined> {
+  return observeRtpStreamStats(participant, source, "inbound-rtp").pipe(
+    map((x) => x as RTCInboundRtpStreamStats | undefined),
+  );
+}
+
+export function observeOutboundRtpStreamStats(
+  participant: Participant,
+  source: Track.Source,
+): Observable<RTCOutboundRtpStreamStats | undefined> {
+  return observeRtpStreamStats(participant, source, "outbound-rtp").pipe(
+    map((x) => x as RTCOutboundRtpStreamStats | undefined),
+  );
+}
+
+function observeRemoteTrackReceivingOkay(
+  participant: Participant,
+  source: Track.Source,
+): Observable<boolean | undefined> {
+  let lastStats: {
+    framesDecoded: number | undefined;
+    framesDropped: number | undefined;
+    framesReceived: number | undefined;
+  } = {
+    framesDecoded: undefined,
+    framesDropped: undefined,
+    framesReceived: undefined,
+  };
+
+  return observeInboundRtpStreamStats(participant, source).pipe(
+    map((stats) => {
+      if (!stats) return undefined;
+      const { framesDecoded, framesDropped, framesReceived } = stats;
+      return {
+        framesDecoded,
+        framesDropped,
+        framesReceived,
+      };
+    }),
     filter((newStats) => !!newStats),
     map((newStats): boolean | undefined => {
       const oldStats = lastStats;
@@ -401,6 +437,13 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   public get local(): boolean {
     return this instanceof LocalUserMediaViewModel;
   }
+
+  public abstract get audioStreamStats(): Observable<
+    RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+  >;
+  public abstract get videoStreamStats(): Observable<
+    RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats | undefined
+  >;
 }
 
 /**
@@ -440,6 +483,26 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   ) {
     super(id, member, participant, encryptionSystem, livekitRoom);
   }
+
+  public audioStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeOutboundRtpStreamStats(p, Track.Source.Microphone);
+    }),
+  );
+
+  public videoStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeOutboundRtpStreamStats(p, Track.Source.Camera);
+    }),
+  );
 }
 
 /**
@@ -519,6 +582,26 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
   public commitLocalVolume(): void {
     this.localVolumeCommit.next();
   }
+
+  public audioStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeInboundRtpStreamStats(p, Track.Source.Microphone);
+    }),
+  );
+
+  public videoStreamStats = combineLatest([
+    this.participant,
+    showConnectionStats.value,
+  ]).pipe(
+    switchMap(([p, enabled]) => {
+      if (!p || !enabled) return of(undefined);
+      return observeInboundRtpStreamStats(p, Track.Source.Camera);
+    }),
+  );
 }
 
 /**

diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx
@@ -86,6 +86,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
     const video = useObservableEagerState(vm.video);
     const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
     const encryptionStatus = useObservableEagerState(vm.encryptionStatus);
+    const audioStreamStats = useObservableEagerState(vm.audioStreamStats);
+    const videoStreamStats = useObservableEagerState(vm.videoStreamStats);
     const audioEnabled = useObservableEagerState(vm.audioEnabled);
     const videoEnabled = useObservableEagerState(vm.videoEnabled);
     const speaking = useObservableEagerState(vm.speaking);
@@ -176,6 +178,8 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         currentReaction={currentReaction}
         raisedHandOnClick={raisedHandOnClick}
         localParticipant={vm.local}
+        audioStreamStats={audioStreamStats}
+        videoStreamStats={videoStreamStats}
         {...props}
       />
     );

diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx
@@ -22,6 +22,7 @@ import { RaisedHandIndicator } from "../reactions/RaisedHandIndicator";
 import { showHandRaisedTimer, useSetting } from "../settings/settings";
 import { type ReactionOption } from "../reactions";
 import { ReactionIndicator } from "../reactions/ReactionIndicator";
+import { RTCConnectionStats } from "../RTCConnectionStats";
 
 interface Props extends ComponentProps<typeof animated.div> {
   className?: string;
@@ -42,6 +43,8 @@ interface Props extends ComponentProps<typeof animated.div> {
   currentReaction?: ReactionOption;
   raisedHandOnClick?: () => void;
   localParticipant: boolean;
+  audioStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
+  videoStreamStats?: RTCInboundRtpStreamStats | RTCOutboundRtpStreamStats;
 }
 
 export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -65,6 +68,8 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       currentReaction,
       raisedHandOnClick,
       localParticipant,
+      audioStreamStats,
+      videoStreamStats,
       ...props
     },
     ref,
@@ -125,6 +130,12 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
               {t("video_tile.waiting_for_media")}
             </div>
           )}
+          {(audioStreamStats || videoStreamStats) && (
+            <RTCConnectionStats
+              audio={audioStreamStats}
+              video={videoStreamStats}
+            />
+          )}
           {/* TODO: Bring this back once encryption status is less broken */}
           {/*encryptionStatus !== EncryptionStatus.Okay && (
             <div className={styles.status}>