refactor

 - still should not work without a fixing upstream LK:
livekit/components-js#1042
livekit/components-js#1043

src/App.tsx

@@ -28,6 +28,7 @@ import { Initializer } from "./initializer";
 import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
+import { ProcessorProvider } from "./livekit/TrackProcessorContext";
 
 const SentryRoute = Sentry.withSentryRouting(Route);
 
@@ -82,27 +83,25 @@ export const App: FC<AppProps> = ({ history }) => {
           <TooltipProvider>
             {loaded ? (
               <Suspense fallback={null}>
-                <ClientProvider>
-                  <MediaDevicesProvider>
-                    <Sentry.ErrorBoundary fallback={errorPage}>
-                      <DisconnectedBanner />
-                      <Switch>
-                        <SentryRoute exact path="/">
-                          <HomePage />
-                        </SentryRoute>
-                        <SentryRoute exact path="/login">
-                          <LoginPage />
-                        </SentryRoute>
-                        <SentryRoute exact path="/register">
-                          <RegisterPage />
-                        </SentryRoute>
-                        <SentryRoute path="*">
-                          <RoomPage />
-                        </SentryRoute>
-                      </Switch>
-                    </Sentry.ErrorBoundary>
-                  </MediaDevicesProvider>
-                </ClientProvider>
+                <Providers>
+                  <Sentry.ErrorBoundary fallback={errorPage}>
+                    <DisconnectedBanner />
+                    <Switch>
+                      <SentryRoute exact path="/">
+                        <HomePage />
+                      </SentryRoute>
+                      <SentryRoute exact path="/login">
+                        <LoginPage />
+                      </SentryRoute>
+                      <SentryRoute exact path="/register">
+                        <RegisterPage />
+                      </SentryRoute>
+                      <SentryRoute path="*">
+                        <RoomPage />
+                      </SentryRoute>
+                    </Switch>
+                  </Sentry.ErrorBoundary>
+                </Providers>
               </Suspense>
             ) : (
               <LoadingView />
@@ -113,3 +112,16 @@ export const App: FC<AppProps> = ({ history }) => {
     </Router>
   );
 };
+
+const Providers: FC<{
+  children: JSX.Element;
+}> = ({ children }) => {
+  // We use this to stack all used providers to not make the App component to verbose
+  return (
+    <ClientProvider>
+      <MediaDevicesProvider>
+        <ProcessorProvider>{children}</ProcessorProvider>
+      </MediaDevicesProvider>
+    </ClientProvider>
+  );
+};

src/livekit/TrackProcessorContext.tsx

@@ -0,0 +1,111 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  BackgroundBlur as backgroundBlur,
+  BackgroundOptions,
+  ProcessorWrapper,
+} from "@livekit/track-processors";
+import {
+  createContext,
+  FC,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+import { logger } from "matrix-js-sdk/src/logger";
+import { LocalVideoTrack } from "livekit-client";
+
+import {
+  backgroundBlur as backgroundBlurSettings,
+  useSetting,
+} from "../settings/settings";
+
+type ProcessorState = {
+  supported: boolean | undefined;
+  processor: undefined | ProcessorWrapper<BackgroundOptions>;
+  /**
+   * Call this method to try to initialize a processor.
+   * This only needs to happen if supported is undefined.
+   * If the backgroundBlur setting is set to true this does not need to be called
+   * and the processorState.supported will update automatically to the correct value.
+   */
+  checkSupported: () => void;
+};
+const ProcessorContext = createContext<ProcessorState | undefined>(undefined);
+
+export const useTrackProcessor = (): ProcessorState | undefined =>
+  useContext(ProcessorContext);
+
+export const useTrackProcessorSync = (
+  videoTrack: LocalVideoTrack | null,
+): void => {
+  const { processor } = useTrackProcessor() || {};
+  useEffect(() => {
+    if (processor && !videoTrack?.getProcessor()) {
+      void videoTrack?.setProcessor(processor);
+    }
+    if (!processor && videoTrack?.getProcessor()) {
+      void videoTrack?.stopProcessor();
+    }
+  }, [processor, videoTrack]);
+};
+
+interface Props {
+  children: JSX.Element;
+}
+export const ProcessorProvider: FC<Props> = ({ children }) => {
+  // The setting the user wants to have
+  const [blurActivated] = useSetting(backgroundBlurSettings);
+
+  // If `ProcessorState.supported` is undefined the user can activate that we want
+  // to have it at least checked (this is useful to show the settings menu properly)
+  // We dont want to try initializing the blur if the user is not even looking at the setting
+  const [shouldCheckSupport, setShouldCheckSupport] = useState(blurActivated);
+
+  // Cache the processor so we only need to initialize it once.
+  const blur = useRef<ProcessorWrapper<BackgroundOptions> | undefined>(
+    undefined,
+  );
+
+  const checkSupported = useCallback(() => {
+    setShouldCheckSupport(true);
+  }, []);
+  // This is the actual state exposed through the context
+  const [processorState, setProcessorState] = useState<ProcessorState>(() => ({
+    supported: false,
+    processor: undefined,
+    checkSupported,
+  }));
+
+  useEffect(() => {
+    if (!shouldCheckSupport) return;
+    try {
+      if (!blur.current) blur.current = backgroundBlur(15, { delegate: "GPU" });
+      setProcessorState({
+        checkSupported,
+        supported: true,
+        processor: blurActivated ? blur.current : undefined,
+      });
+    } catch (e) {
+      setProcessorState({
+        checkSupported,
+        supported: false,
+        processor: undefined,
+      });
+      logger.error("disable background blur", e);
+    }
+  }, [blurActivated, checkSupported, shouldCheckSupport]);
+
+  return (
+    <ProcessorContext.Provider value={processorState}>
+      {children}
+    </ProcessorContext.Provider>
+  );
+};

src/livekit/useLiveKit.ts

@@ -9,17 +9,15 @@ import {
   ConnectionState,
   E2EEOptions,
   ExternalE2EEKeyProvider,
-  LocalTrackPublication,
+  LocalVideoTrack,
   Room,
-  RoomEvent,
   RoomOptions,
   Track,
 } from "livekit-client";
 import { useEffect, useMemo, useRef } from "react";
 import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import { logger } from "matrix-js-sdk/src/logger";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
-import { BackgroundBlur as backgroundBlur } from "@livekit/track-processors";
 
 import { defaultLiveKitOptions } from "./options";
 import { SFUConfig } from "./openIDSFU";
@@ -29,15 +27,18 @@ import {
   MediaDevices,
   useMediaDevices,
 } from "./MediaDevicesContext";
-import { backgroundBlur as backgroundBlurSettings } from "../settings/settings";
 import {
   ECConnectionState,
   useECConnectionState,
 } from "./useECConnectionState";
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
 import { E2eeType } from "../e2ee/e2eeType";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
-import { useSetting } from "../settings/settings";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { useInitial } from "../useInitial";
 
 interface UseLivekitResult {
   livekitRoom?: Room;
@@ -83,22 +84,16 @@ export function useLiveKit(
   const initialMuteStates = useRef<MuteStates>(muteStates);
   const devices = useMediaDevices();
   const initialDevices = useRef<MediaDevices>(devices);
-  const blur = useMemo(() => {
-    let b = undefined;
-    try {
-      b = backgroundBlur(15, { delegate: "GPU" });
-    } catch (e) {
-      logger.error("disable background blur", e);
-    }
-    return b;
-  }, []);
+
+  const { processor } = useTrackProcessor() || {};
+  const initialProcessor = useInitial(() => processor);
   const roomOptions = useMemo(
     (): RoomOptions => ({
       ...defaultLiveKitOptions,
       videoCaptureDefaults: {
         ...defaultLiveKitOptions.videoCaptureDefaults,
         deviceId: initialDevices.current.videoInput.selectedId,
-        processor: blur,
+        processor: initialProcessor,
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
@@ -109,7 +104,7 @@ export function useLiveKit(
       },
       e2ee: e2eeOptions,
     }),
-    [blur, e2eeOptions],
+    [e2eeOptions, initialProcessor],
   );
 
   // Store if audio/video are currently updating. If to prohibit unnecessary calls
@@ -134,6 +129,15 @@ export function useLiveKit(
     return r;
   }, [roomOptions, e2eeSystem]);
 
+  const videoTrack = useMemo(
+    () =>
+      Array.from(room.localParticipant.videoTrackPublications.values()).find(
+        (v) => v.source === Track.Source.Camera,
+      )?.track as LocalVideoTrack | null,
+    [room.localParticipant.videoTrackPublications],
+  );
+  useTrackProcessorSync(videoTrack);
+
   const connectionState = useECConnectionState(
     {
       deviceId: initialDevices.current.audioInput.selectedId,
@@ -143,58 +147,6 @@ export function useLiveKit(
     sfuConfig,
   );
 
-  const [showBackgroundBlur] = useSetting(backgroundBlurSettings);
-  const videoTrackPromise = useRef<
-    undefined | Promise<LocalTrackPublication | undefined>
-  >(undefined);
-
-  useEffect(() => {
-    // Don't even try if we cannot blur on this platform
-    if (!blur) return;
-    if (!room || videoTrackPromise.current) return;
-    const update = async (): Promise<void> => {
-      let publishCallback: undefined | ((track: LocalTrackPublication) => void);
-      videoTrackPromise.current = new Promise<
-        LocalTrackPublication | undefined
-      >((resolve) => {
-        const videoTrack = Array.from(
-          room.localParticipant.videoTrackPublications.values(),
-        ).find((v) => v.source === Track.Source.Camera);
-        if (videoTrack) {
-          resolve(videoTrack);
-        }
-        publishCallback = (videoTrack: LocalTrackPublication): void => {
-          if (videoTrack.source === Track.Source.Camera) {
-            resolve(videoTrack);
-          }
-        };
-        room.on(RoomEvent.LocalTrackPublished, publishCallback);
-      });
-
-      const videoTrack = await videoTrackPromise.current;
-
-      if (publishCallback)
-        room.off(RoomEvent.LocalTrackPublished, publishCallback);
-
-      if (videoTrack !== undefined) {
-        if (
-          showBackgroundBlur &&
-          videoTrack.track?.getProcessor()?.name !== "background-blur"
-        ) {
-          logger.info("Blur: set blur");
-
-          void videoTrack.track?.setProcessor(blur);
-        } else if (
-          videoTrack.track?.getProcessor()?.name === "background-blur"
-        ) {
-          void videoTrack.track?.stopProcessor();
-        }
-      }
-      videoTrackPromise.current = undefined;
-    };
-    void update();
-  }, [blur, room, showBackgroundBlur]);
-
   useEffect(() => {
     // Sync the requested mute states with LiveKit's mute states. We do it this
     // way around rather than using LiveKit as the source of truth, so that the
@@ -261,13 +213,15 @@ export function useLiveKit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
+                  room.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
               case MuteDevice.Camera:
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
+                  room.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;