From ba6b1ab80276e37f609229d915e818d3b775b344 Mon Sep 17 00:00:00 2001
From: David Zhao <dz@livekit.io>
Date: Tue, 12 Nov 2024 00:03:35 -0800
Subject: [PATCH] Avoid recording room composite before video is decoded

It's likely to receive delta frames that are not decodable in
the start. This would produce an undesirable black screen in the beginning of the recording.
---
 template-default/src/Room.tsx | 63 +++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 21 deletions(-)

diff --git a/template-default/src/Room.tsx b/template-default/src/Room.tsx
index b55c4f4a..4b26ad39 100644
--- a/template-default/src/Room.tsx
+++ b/template-default/src/Room.tsx
@@ -23,7 +23,7 @@ import {
   useTracks,
 } from '@livekit/components-react';
 import EgressHelper from '@livekit/egress-sdk';
-import { ConnectionState, RoomEvent, Track } from 'livekit-client';
+import { ConnectionState, Track } from 'livekit-client';
 import { ReactElement, useEffect, useState } from 'react';
 import SingleSpeakerLayout from './SingleSpeakerLayout';
 import SpeakerLayout from './SpeakerLayout';
@@ -53,38 +53,59 @@ interface CompositeTemplateProps {
 
 function CompositeTemplate({ layout: initialLayout }: CompositeTemplateProps) {
   const room = useRoomContext();
-  const [layout, setLayout] = useState(initialLayout);
+  const [layout] = useState(initialLayout);
   const [hasScreenShare, setHasScreenShare] = useState(false);
   const screenshareTracks = useTracks([Track.Source.ScreenShare], {
     onlySubscribed: true,
   });
 
   useEffect(() => {
-    if (room) {
-      EgressHelper.setRoom(room);
-
-      // Egress layout can change on the fly, we can react to the new layout
-      // here.
-      EgressHelper.onLayoutChanged((newLayout) => {
-        setLayout(newLayout);
-      });
-
-      // start recording when there's already a track published
-      let hasTrack = false;
+    // determines when to start recording
+    // the algorithm used is:
+    // * if there are video tracks published, wait for frames to be decoded
+    // * if there are no video tracks published, start immediately
+    // * if it's been more than 10s, record as long as there are tracks subscribed
+    const startTime = Date.now();
+    const interval = setInterval(async () => {
+      let shouldStartRecording = false;
+      let hasVideoTracks = false;
+      let hasSubscribedTracks = false;
+      let hasDecodedFrames = false;
       for (const p of Array.from(room.remoteParticipants.values())) {
-        if (p.trackPublications.size > 0) {
-          hasTrack = true;
-          break;
+        for (const pub of Array.from(p.trackPublications.values())) {
+          if (pub.isSubscribed) {
+            hasSubscribedTracks = true;
+          }
+          if (pub.kind === Track.Kind.Video) {
+            hasVideoTracks = true;
+            if (pub.videoTrack) {
+              const stats = await pub.videoTrack.getRTCStatsReport();
+              if (stats) {
+                hasDecodedFrames = Array.from(stats).some(
+                  (item) => item[1].type === 'inbound-rtp' && item[1].framesDecoded > 0,
+                );
+              }
+            }
+          }
         }
       }
 
-      if (hasTrack) {
+      const timeDelta = Date.now() - startTime;
+      if (hasDecodedFrames) {
+        shouldStartRecording = true;
+      } else if (!hasVideoTracks && hasSubscribedTracks && timeDelta > 500) {
+        // adding a small timeout to ensure video tracks has a chance to be published
+        shouldStartRecording = true;
+      } else if (timeDelta > 10000 && hasSubscribedTracks) {
+        shouldStartRecording = true;
+      }
+
+      if (shouldStartRecording) {
         EgressHelper.startRecording();
-      } else {
-        room.once(RoomEvent.TrackSubscribed, () => EgressHelper.startRecording());
+        clearInterval(interval);
       }
-    }
-  }, [room]);
+    }, 100);
+  }, []);
 
   useEffect(() => {
     if (screenshareTracks.length > 0 && screenshareTracks[0].publication) {