From cf39d904666d8a8de76f7eefe2db8a9dd554b06a Mon Sep 17 00:00:00 2001 From: David Eberlein Date: Wed, 20 Nov 2024 18:09:56 +0100 Subject: [PATCH 1/4] Add support for detecting video element visibility in Document PiP Drop opacity check as the intersection observer is not triggered if an element changes opacity and the check is flawed anyway as opacity of parents isn't considered. --- src/room/track/RemoteVideoTrack.ts | 52 ++++++++++++++++++---------- src/type-polyfills/document-pip.d.ts | 11 ++++++ 2 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 src/type-polyfills/document-pip.d.ts diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 1268d95efb..42604adc6d 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -331,7 +331,7 @@ class HTMLElementInfo implements ElementInfo { constructor(element: HTMLMediaElement, visible?: boolean) { this.element = element; this.isIntersecting = visible ?? isElementInViewport(element); - this.isPiP = isWeb() && document.pictureInPictureElement === element; + this.isPiP = isWeb() && isElementInPiP(element); this.visibilityChangedAt = 0; } @@ -346,7 +346,7 @@ class HTMLElementInfo implements ElementInfo { observe() { // make sure we update the current visible state once we start to observe this.isIntersecting = isElementInViewport(this.element); - this.isPiP = document.pictureInPictureElement === this.element; + this.isPiP = isElementInPiP(this.element); (this.element as ObservableMediaElement).handleResize = () => { this.handleResize?.(); @@ -357,24 +357,28 @@ class HTMLElementInfo implements ElementInfo { getResizeObserver().observe(this.element); (this.element as HTMLVideoElement).addEventListener('enterpictureinpicture', this.onEnterPiP); (this.element as HTMLVideoElement).addEventListener('leavepictureinpicture', this.onLeavePiP); + window.documentPictureInPicture?.addEventListener('enter', this.onEnterPiP); + window.documentPictureInPicture?.window?.addEventListener('pagehide', this.onLeavePiP); } private onVisibilityChanged = (entry: IntersectionObserverEntry) => { const { target, isIntersecting } = entry; if (target === this.element) { this.isIntersecting = isIntersecting; + this.isPiP = isElementInPiP(this.element); this.visibilityChangedAt = Date.now(); this.handleVisibilityChanged?.(); } }; private onEnterPiP = () => { - this.isPiP = true; + window.documentPictureInPicture?.window?.addEventListener('pagehide', this.onLeavePiP); + this.isPiP = isElementInPiP(this.element); this.handleVisibilityChanged?.(); }; private onLeavePiP = () => { - this.isPiP = false; + this.isPiP = isElementInPiP(this.element); this.handleVisibilityChanged?.(); }; @@ -382,24 +386,37 @@ class HTMLElementInfo implements ElementInfo { getIntersectionObserver()?.unobserve(this.element); getResizeObserver()?.unobserve(this.element); (this.element as HTMLVideoElement).removeEventListener( - 'enterpictureinpicture', - this.onEnterPiP, + 'enterpictureinpicture', + this.onEnterPiP, ); (this.element as HTMLVideoElement).removeEventListener( - 'leavepictureinpicture', - this.onLeavePiP, + 'leavepictureinpicture', + this.onLeavePiP, ); + window.documentPictureInPicture?.removeEventListener('enter', this.onEnterPiP); + window.documentPictureInPicture?.window?.removeEventListener('pagehide', this.onLeavePiP); } } -// does not account for occlusion by other elements -function isElementInViewport(el: HTMLElement) { +function isElementInPiP(el: HTMLElement) { + // Simple video PiP + if(document.pictureInPictureElement === el) + return true; + // Document PiP + if(window.documentPictureInPicture?.window) + return isElementInViewport(el, window.documentPictureInPicture?.window); + return false; +} + +// does not account for occlusion by other elements or opacity property +function isElementInViewport(el: HTMLElement, win?: Window) { + const viewportWindow = win || window; let top = el.offsetTop; let left = el.offsetLeft; const width = el.offsetWidth; const height = el.offsetHeight; const { hidden } = el; - const { opacity, display } = getComputedStyle(el); + const { display } = getComputedStyle(el); while (el.offsetParent) { el = el.offsetParent as HTMLElement; @@ -408,12 +425,11 @@ function isElementInViewport(el: HTMLElement) { } return ( - top < window.pageYOffset + window.innerHeight && - left < window.pageXOffset + window.innerWidth && - top + height > window.pageYOffset && - left + width > window.pageXOffset && - !hidden && - (opacity !== '' ? parseFloat(opacity) > 0 : true) && - display !== 'none' + top < viewportWindow.pageYOffset + viewportWindow.innerHeight && + left < viewportWindow.pageXOffset + viewportWindow.innerWidth && + top + height > viewportWindow.pageYOffset && + left + width > viewportWindow.pageXOffset && + !hidden && + display !== 'none' ); } diff --git a/src/type-polyfills/document-pip.d.ts b/src/type-polyfills/document-pip.d.ts new file mode 100644 index 0000000000..fde01168d2 --- /dev/null +++ b/src/type-polyfills/document-pip.d.ts @@ -0,0 +1,11 @@ +interface Window { + /** + * Currently only available in Chromium based browsers: + * https://developer.mozilla.org/en-US/docs/Web/API/DocumentPictureInPicture + */ + documentPictureInPicture?: DocumentPictureInPicture; +} + +interface DocumentPictureInPicture extends EventTarget { + window?: Window +} \ No newline at end of file From 7b7774a9a8a66b07170b345f42e1d70425a99ab6 Mon Sep 17 00:00:00 2001 From: David Eberlein Date: Thu, 21 Nov 2024 22:05:07 +0100 Subject: [PATCH 2/4] add document pip to demo page --- examples/demo/demo.ts | 62 +++++++++++++++++++++++----- examples/demo/index.html | 9 ++++ src/room/track/RemoteVideoTrack.ts | 27 ++++++------ src/type-polyfills/document-pip.d.ts | 15 +++---- 4 files changed, 81 insertions(+), 32 deletions(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 5ac892b205..79640eb833 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -278,6 +278,43 @@ const appActions = { await currentRoom.setE2EEEnabled(!currentRoom.isE2EEEnabled); }, + togglePiP: async () => { + if (window.documentPictureInPicture?.window) { + window.documentPictureInPicture?.window.close(); + return; + } + + const pipWindow = await window.documentPictureInPicture?.requestWindow(); + // Copy style sheets over from the initial document + // so that the views look the same. + [...document.styleSheets].forEach((styleSheet) => { + try { + const cssRules = [...styleSheet.cssRules].map((rule) => rule.cssText).join(''); + const style = document.createElement('style'); + style.textContent = cssRules; + pipWindow.document.head.appendChild(style); + } catch (e) { + const link = document.createElement('link'); + link.rel = 'stylesheet'; + link.type = styleSheet.type; + link.media = styleSheet.media; + link.href = styleSheet.href; + pipWindow.document.head.appendChild(link); + } + }); + // Move participant videos to the Picture-in-Picture window + const participantsArea = $('participants-area'); + const pipParticipantsArea = document.createElement('div'); + pipParticipantsArea.id = 'participants-area'; + pipWindow.document.body.append(pipParticipantsArea); + [...participantsArea.children].forEach((child) => pipParticipantsArea.append(child)); + + // Move participant videos back when the Picture-in-Picture window closes. + pipWindow.addEventListener('pagehide', (event) => { + [...pipParticipantsArea.children].forEach((child) => participantsArea.append(child)); + }); + }, + ratchetE2EEKey: async () => { if (!currentRoom || !currentRoom.options.e2ee) { return; @@ -525,10 +562,12 @@ function appendLog(...args: any[]) { // updates participant UI function renderParticipant(participant: Participant, remove: boolean = false) { - const container = $('participants-area'); + const container = + window.documentPictureInPicture?.window?.document.querySelector('#participants-area') || + $('participants-area'); if (!container) return; const { identity } = participant; - let div = $(`participant-${identity}`); + let div = container.querySelector(`#participant-${identity}`); if (!div && !remove) { div = document.createElement('div'); div.id = `participant-${identity}`; @@ -570,8 +609,8 @@ function renderParticipant(participant: Participant, remove: boolean = false) { updateVideoSize(videoElm!, sizeElm!); }; } - const videoElm = $(`video-${identity}`); - const audioELm = $(`audio-${identity}`); + const videoElm = container.querySelector(`#video-${identity}`); + const audioELm = container.querySelector(`#audio-${identity}`); if (remove) { div?.remove(); if (videoElm) { @@ -586,12 +625,12 @@ function renderParticipant(participant: Participant, remove: boolean = false) { } // update properties - $(`name-${identity}`)!.innerHTML = participant.identity; + container.querySelector(`#name-${identity}`)!.innerHTML = participant.identity; if (participant instanceof LocalParticipant) { - $(`name-${identity}`)!.innerHTML += ' (you)'; + container.querySelector(`#name-${identity}`)!.innerHTML += ' (you)'; } - const micElm = $(`mic-${identity}`)!; - const signalElm = $(`signal-${identity}`)!; + const micElm = container.querySelector(`#mic-${identity}`)!; + const signalElm = container.querySelector(`#signal-${identity}`)!; const cameraPub = participant.getTrackPublication(Track.Source.Camera); const micPub = participant.getTrackPublication(Track.Source.Microphone); if (participant.isSpeaking) { @@ -601,7 +640,7 @@ function renderParticipant(participant: Participant, remove: boolean = false) { } if (participant instanceof RemoteParticipant) { - const volumeSlider = $(`volume-${identity}`); + const volumeSlider = container.querySelector(`#volume-${identity}`); volumeSlider.addEventListener('input', (ev) => { participant.setVolume(Number.parseFloat((ev.target as HTMLInputElement).value)); }); @@ -630,7 +669,7 @@ function renderParticipant(participant: Participant, remove: boolean = false) { cameraPub?.videoTrack?.attach(videoElm); } else { // clear information display - $(`size-${identity}`)!.innerHTML = ''; + container.querySelector(`#size-${identity}`)!.innerHTML = ''; if (cameraPub?.videoTrack) { // detach manually whenever possible cameraPub.videoTrack?.detach(videoElm); @@ -659,7 +698,7 @@ function renderParticipant(participant: Participant, remove: boolean = false) { micElm.innerHTML = ''; } - const e2eeElm = $(`e2ee-${identity}`)!; + const e2eeElm = container.querySelector(`#e2ee-${identity}`)!; if (participant.isEncrypted) { e2eeElm.className = 'e2ee-on'; e2eeElm.innerHTML = ''; @@ -795,6 +834,7 @@ function setButtonsForState(connected: boolean) { 'toggle-video-button', 'toggle-audio-button', 'share-screen-button', + 'toggle-pip-button', 'disconnect-ws-button', 'disconnect-room-button', 'flip-video-button', diff --git a/examples/demo/index.html b/examples/demo/index.html index 02c04a1f38..6a810a3eba 100644 --- a/examples/demo/index.html +++ b/examples/demo/index.html @@ -132,6 +132,15 @@

Livekit Sample App

> Share Screen +