Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(svc-codec): VP9 and AV1 support #1434

Merged
merged 40 commits into from
Oct 16, 2024
Merged
Show file tree
Hide file tree
Changes from 35 commits
Commits
Show all changes
40 commits
Select commit Hold shift + click to select a range
df1f240
feat(vp9): experimental VP9 support
oliverlaz Jul 4, 2024
6534062
fix: missing type
oliverlaz Jul 4, 2024
fe7f0c4
fix: publish 30fps for all qualities
oliverlaz Jul 5, 2024
d45991f
fix: publish 30fps for all qualities
oliverlaz Jul 5, 2024
2d9d812
enable av1 as well
su225 Jul 23, 2024
c9d292e
fix: apply correct formatting
oliverlaz Jul 23, 2024
86aacd7
Merge branch 'main' into vp9-publish
oliverlaz Sep 5, 2024
7436760
Merge branch 'refs/heads/main' into vp9-publish
oliverlaz Oct 1, 2024
a4c8710
fix: improve video layer assignment
oliverlaz Oct 1, 2024
c336ae2
chore: add test
oliverlaz Oct 1, 2024
48d7031
Merge branch 'refs/heads/main' into vp9-publish
oliverlaz Oct 8, 2024
d9b5f2e
feat: handle the `scalabilityMode` param in changePublishQuality
oliverlaz Oct 8, 2024
2d10990
feat: for SVC codecs send one layer but announce three
oliverlaz Oct 8, 2024
f2d46c1
feat: remap `f` to `q`
oliverlaz Oct 8, 2024
fef5dad
feat: improve bitrates
oliverlaz Oct 8, 2024
bca74ea
feat: improve bitrates
oliverlaz Oct 8, 2024
12a9939
Merge branch 'main' into vp9-publish
oliverlaz Oct 10, 2024
ece6370
fix: separate svc from simulcast handling
oliverlaz Oct 10, 2024
ce5ffaa
tests for changePublishQuality
oliverlaz Oct 10, 2024
da0e4fb
tests for changePublishQuality
oliverlaz Oct 10, 2024
9770def
chore: simplify type
oliverlaz Oct 10, 2024
89afbb1
fix: use boolean
oliverlaz Oct 11, 2024
01a5cec
feat: move bitrate lookup to the SDK
oliverlaz Oct 14, 2024
2b1b7fc
Merge branch 'main' into vp9-publish
oliverlaz Oct 14, 2024
d1f4b5e
feat: allow guest logins in Pronto
oliverlaz Oct 15, 2024
a9427af
chore: update to rn-webrtc 124
santhoshvai Oct 15, 2024
c4e828e
feat: optimal codec selection, refactored Publisher for improved read…
oliverlaz Oct 15, 2024
696a4ef
chore: moved `updatePublishOptions` to the `call` instance
oliverlaz Oct 15, 2024
7071ad7
feat: add `forceCodec` option
oliverlaz Oct 15, 2024
850301b
chore: adjust tests
oliverlaz Oct 15, 2024
fd4c0c5
chore: reorganize Publisher
oliverlaz Oct 16, 2024
94ebaaa
fix: use vp8 as a default codec for Android
oliverlaz Oct 16, 2024
a516f47
chore: update roadmaps
oliverlaz Oct 16, 2024
449438d
Merge branch 'main' into vp9-publish
oliverlaz Oct 16, 2024
a1a9418
chore: missing checksums
oliverlaz Oct 16, 2024
5a15195
chore: backport the fixes of #1437
oliverlaz Oct 16, 2024
41a9a5b
chore: update roadmap
oliverlaz Oct 16, 2024
77dc4a3
Merge branch 'main' into vp9-publish
oliverlaz Oct 16, 2024
2c44861
chore: restore auto-release
oliverlaz Oct 16, 2024
6b84d4b
chore: restore auto-release
oliverlaz Oct 16, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 21 additions & 38 deletions packages/client/src/Call.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,7 @@ import {
VideoTrackType,
} from './types';
import { BehaviorSubject, Subject, takeWhile } from 'rxjs';
import {
ReconnectDetails,
VideoLayerSetting,
} from './gen/video/sfu/event/events';
import { ReconnectDetails } from './gen/video/sfu/event/events';
import {
ClientDetails,
TrackType,
Expand Down Expand Up @@ -202,6 +199,7 @@ export class Call {
*/
private readonly dispatcher = new Dispatcher();

private publishOptions?: PublishOptions;
private statsReporter?: StatsReporter;
private sfuStatsReporter?: SfuStatsReporter;
private dropTimeout: ReturnType<typeof setTimeout> | undefined;
Expand Down Expand Up @@ -1292,19 +1290,12 @@ export class Call {
break;
case TrackType.VIDEO:
const videoStream = this.camera.state.mediaStream;
if (videoStream) {
await this.publishVideoStream(
videoStream,
this.camera.publishOptions,
);
}
if (videoStream) await this.publishVideoStream(videoStream);
break;
case TrackType.SCREEN_SHARE:
const screenShareStream = this.screenShare.state.mediaStream;
if (screenShareStream) {
await this.publishScreenShareStream(screenShareStream, {
screenShareSettings: this.screenShare.getSettings(),
});
await this.publishScreenShareStream(screenShareStream);
}
break;
// screen share audio can't exist without a screen share, so we handle it there
Expand Down Expand Up @@ -1336,12 +1327,8 @@ export class Call {
* The previous video stream will be stopped.
*
* @param videoStream the video stream to publish.
* @param opts the options to use when publishing the stream.
*/
publishVideoStream = async (
videoStream: MediaStream,
opts: PublishOptions = {},
) => {
publishVideoStream = async (videoStream: MediaStream) => {
if (!this.sfuClient) throw new Error(`Call not joined yet.`);
// joining is in progress, and we should wait until the client is ready
await this.sfuClient.joinTask;
Expand All @@ -1363,7 +1350,7 @@ export class Call {
videoStream,
videoTrack,
TrackType.VIDEO,
opts,
this.publishOptions,
);
};

Expand Down Expand Up @@ -1407,12 +1394,8 @@ export class Call {
* The previous screen-share stream will be stopped.
*
* @param screenShareStream the screen-share stream to publish.
* @param opts the options to use when publishing the stream.
*/
publishScreenShareStream = async (
screenShareStream: MediaStream,
opts: PublishOptions = {},
) => {
publishScreenShareStream = async (screenShareStream: MediaStream) => {
if (!this.sfuClient) throw new Error(`Call not joined yet.`);
// joining is in progress, and we should wait until the client is ready
await this.sfuClient.joinTask;
Expand All @@ -1431,6 +1414,9 @@ export class Call {
if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE)) {
this.trackPublishOrder.push(TrackType.SCREEN_SHARE);
}
const opts: PublishOptions = {
screenShareSettings: this.screenShare.getSettings(),
};
await this.publisher.publishStream(
screenShareStream,
screenShareTrack,
Expand Down Expand Up @@ -1467,6 +1453,16 @@ export class Call {
await this.publisher?.unpublishStream(trackType, stopTrack);
};

/**
* Updates the preferred publishing options
*
* @internal
* @param options the options to use.
*/
updatePublishOptions(options: PublishOptions) {
this.publishOptions = { ...this.publishOptions, ...options };
}

/**
* Notifies the SFU that a noise cancellation process has started.
*
Expand Down Expand Up @@ -1529,16 +1525,6 @@ export class Call {
return this.state.setSortParticipantsBy(criteria);
};

/**
* Updates the list of video layers to publish.
*
* @internal
* @param enabledLayers the list of layers to enable.
*/
updatePublishQuality = async (enabledLayers: VideoLayerSetting[]) => {
return this.publisher?.updateVideoPublishQuality(enabledLayers);
};

/**
* Sends a reaction to the other call participants.
*
Expand Down Expand Up @@ -2098,10 +2084,7 @@ export class Call {
this.camera.state.mediaStream &&
!this.publisher?.isPublishing(TrackType.VIDEO)
) {
await this.publishVideoStream(
this.camera.state.mediaStream,
this.camera.publishOptions,
);
await this.publishVideoStream(this.camera.state.mediaStream);
}

// Start camera if backend config specifies, and there is no local setting
Expand Down
42 changes: 4 additions & 38 deletions packages/client/src/devices/CameraManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,14 @@ import { CameraDirection, CameraManagerState } from './CameraManagerState';
import { InputMediaDeviceManager } from './InputMediaDeviceManager';
import { getVideoDevices, getVideoStream } from './devices';
import { TrackType } from '../gen/video/sfu/models/models';
import { PreferredCodec, PublishOptions } from '../types';
import { PreferredCodec } from '../types';

export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
private targetResolution = {
width: 1280,
height: 720,
};

/**
* The options to use when publishing the video stream.
*
* @internal
*/
publishOptions: PublishOptions | undefined;

/**
* Constructs a new CameraManager.
*
Expand Down Expand Up @@ -86,38 +79,11 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
* Sets the preferred codec for encoding the video.
*
* @internal internal use only, not part of the public API.
* @deprecated use {@link call.updatePublishOptions} instead.
* @param codec the codec to use for encoding the video.
*/
setPreferredCodec(codec: PreferredCodec | undefined) {
this.updatePublishOptions({ preferredCodec: codec });
}

/**
* Updates the preferred publish options for the video stream.
*
* @internal
* @param options the options to use.
*/
updatePublishOptions(options: PublishOptions) {
this.publishOptions = { ...this.publishOptions, ...options };
}

/**
* Returns the capture resolution of the camera.
*/
getCaptureResolution() {
const { mediaStream } = this.state;
if (!mediaStream) return;

const [videoTrack] = mediaStream.getVideoTracks();
if (!videoTrack) return;

const settings = videoTrack.getSettings();
return {
width: settings.width,
height: settings.height,
frameRate: settings.frameRate,
};
this.call.updatePublishOptions({ preferredCodec: codec });
}

protected getDevices(): Observable<MediaDeviceInfo[]> {
Expand All @@ -139,7 +105,7 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
}

protected publishStream(stream: MediaStream): Promise<void> {
return this.call.publishVideoStream(stream, this.publishOptions);
return this.call.publishVideoStream(stream);
}

protected stopPublishStream(stopTracks: boolean): Promise<void> {
Expand Down
4 changes: 1 addition & 3 deletions packages/client/src/devices/ScreenShareManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,7 @@ export class ScreenShareManager extends InputMediaDeviceManager<
}

protected publishStream(stream: MediaStream): Promise<void> {
return this.call.publishScreenShareStream(stream, {
screenShareSettings: this.state.settings,
});
return this.call.publishScreenShareStream(stream);
}

protected async stopPublishStream(stopTracks: boolean): Promise<void> {
Expand Down
15 changes: 0 additions & 15 deletions packages/client/src/devices/__tests__/CameraManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,21 +82,6 @@ describe('CameraManager', () => {

expect(manager['call'].publishVideoStream).toHaveBeenCalledWith(
manager.state.mediaStream,
undefined,
);
});

it('publish stream with preferred codec', async () => {
manager['call'].state.setCallingState(CallingState.JOINED);
manager.setPreferredCodec('h264');

await manager.enable();

expect(manager['call'].publishVideoStream).toHaveBeenCalledWith(
manager.state.mediaStream,
{
preferredCodec: 'h264',
},
);
});

Expand Down
14 changes: 0 additions & 14 deletions packages/client/src/devices/__tests__/ScreenShareManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,20 +115,6 @@ describe('ScreenShareManager', () => {
await manager.enable();
expect(call.publishScreenShareStream).toHaveBeenCalledWith(
manager.state.mediaStream,
{ screenShareSettings: undefined },
);
});

it('publishes screen share stream with settings', async () => {
const call = manager['call'];
call.state.setCallingState(CallingState.JOINED);

manager.setSettings({ maxFramerate: 15, maxBitrate: 1000 });

await manager.enable();
expect(call.publishScreenShareStream).toHaveBeenCalledWith(
manager.state.mediaStream,
{ screenShareSettings: { maxFramerate: 15, maxBitrate: 1000 } },
);
});

Expand Down
2 changes: 0 additions & 2 deletions packages/client/src/events/callEventHandlers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import {
watchCallEnded,
watchCallGrantsUpdated,
watchCallRejected,
watchChangePublishQuality,
watchConnectionQualityChanged,
watchDominantSpeakerChanged,
watchLiveEnded,
Expand Down Expand Up @@ -46,7 +45,6 @@ export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => {

watchLiveEnded(dispatcher, call),
watchSfuErrorReports(dispatcher),
watchChangePublishQuality(dispatcher, call),
watchConnectionQualityChanged(dispatcher, state),
watchParticipantCountChanged(dispatcher, state),

Expand Down
16 changes: 0 additions & 16 deletions packages/client/src/events/internal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,22 +10,6 @@ import {
} from '../gen/video/sfu/models/models';
import { OwnCapability } from '../gen/coordinator';

/**
* An event responder which handles the `changePublishQuality` event.
*/
export const watchChangePublishQuality = (
dispatcher: Dispatcher,
call: Call,
) => {
return dispatcher.on('changePublishQuality', (e) => {
const { videoSenders } = e;
videoSenders.forEach((videoSender) => {
const { layers } = videoSender;
call.updatePublishQuality(layers.filter((l) => l.active));
});
});
};

export const watchConnectionQualityChanged = (
dispatcher: Dispatcher,
state: CallState,
Expand Down
Loading
Loading