Skip to content

Commit

Permalink
screen sharing for desktop. (#135)
Browse files Browse the repository at this point in the history 
* screen sharing for desktop (WIP).

* use git repo for testing.

* update.

* Add sourceId/frameRate to ScreenShareCaptureOptions.

* Compatible with flutter web.

* Rendering of camera and screenshare video tracks at same time.

* Remote screen sharing is shown first, and fixed sub/unsub button for screenshare widget.

* resolved conflict.

* modify flutter-webrtc to pub version.

* remove unused import.

* chore: Common params for Camera and ScreenShare.
  • Loading branch information
@cloudwebrtc
cloudwebrtc authored Aug 2, 2022
1 parent c772271 commit a554d92
Show file tree
Hide file tree
Showing 13 changed files with 533 additions and 103 deletions.
77 changes: 54 additions & 23 deletions example/lib/pages/room.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import 'package:livekit_client/livekit_client.dart';
import '../exts.dart';
import '../widgets/controls.dart';
import '../widgets/participant.dart';
import '../widgets/participant_info.dart';

class RoomPage extends StatefulWidget {
//
Expand All @@ -25,7 +26,7 @@ class RoomPage extends StatefulWidget {

class _RoomPageState extends State<RoomPage> {
//
List<Participant> participants = [];
List<ParticipantTrack> participantTracks = [];
EventsListener<RoomEvent> get _listener => widget.listener;
bool get fastConnection => widget.room.engine.fastConnectOptions != null;
@override
Expand Down Expand Up @@ -57,6 +58,8 @@ class _RoomPageState extends State<RoomPage> {
WidgetsBindingCompatible.instance
?.addPostFrameCallback((timeStamp) => Navigator.pop(context));
})
..on<LocalTrackPublishedEvent>((_) => _sortParticipants())
..on<LocalTrackUnpublishedEvent>((_) => _sortParticipants())
..on<DataReceivedEvent>((event) {
String decoded = 'Failed to decode';
try {
Expand Down Expand Up @@ -90,47 +93,74 @@ class _RoomPageState extends State<RoomPage> {
}

void _sortParticipants() {
List<Participant> participants = [];
participants.addAll(widget.room.participants.values);
List<ParticipantTrack> userMediaTracks = [];
List<ParticipantTrack> screenTracks = [];
for (var participant in widget.room.participants.values) {
for (var t in participant.videoTracks) {
if (t.isScreenShare) {
screenTracks.add(ParticipantTrack(
participant: participant,
videoTrack: t.track,
isScreenShare: true,
));
} else {
userMediaTracks.add(ParticipantTrack(
participant: participant,
videoTrack: t.track,
isScreenShare: false,
));
}
}
}
// sort speakers for the grid
participants.sort((a, b) {
userMediaTracks.sort((a, b) {
// loudest speaker first
if (a.isSpeaking && b.isSpeaking) {
if (a.audioLevel > b.audioLevel) {
if (a.participant.isSpeaking && b.participant.isSpeaking) {
if (a.participant.audioLevel > b.participant.audioLevel) {
return -1;
} else {
return 1;
}
}

// last spoken at
final aSpokeAt = a.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
final bSpokeAt = b.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
final aSpokeAt = a.participant.lastSpokeAt?.millisecondsSinceEpoch ?? 0;
final bSpokeAt = b.participant.lastSpokeAt?.millisecondsSinceEpoch ?? 0;

if (aSpokeAt != bSpokeAt) {
return aSpokeAt > bSpokeAt ? -1 : 1;
}

// video on
if (a.hasVideo != b.hasVideo) {
return a.hasVideo ? -1 : 1;
if (a.participant.hasVideo != b.participant.hasVideo) {
return a.participant.hasVideo ? -1 : 1;
}

// joinedAt
return a.joinedAt.millisecondsSinceEpoch -
b.joinedAt.millisecondsSinceEpoch;
return a.participant.joinedAt.millisecondsSinceEpoch -
b.participant.joinedAt.millisecondsSinceEpoch;
});

final localParticipant = widget.room.localParticipant;
if (localParticipant != null) {
if (participants.length > 1) {
participants.insert(1, localParticipant);
} else {
participants.add(localParticipant);
final localParticipantTracks = widget.room.localParticipant?.videoTracks;
if (localParticipantTracks != null) {
for (var t in localParticipantTracks) {
if (t.isScreenShare) {
screenTracks.add(ParticipantTrack(
participant: widget.room.localParticipant!,
videoTrack: t.track,
isScreenShare: true,
));
} else {
userMediaTracks.add(ParticipantTrack(
participant: widget.room.localParticipant!,
videoTrack: t.track,
isScreenShare: false,
));
}
}
}
setState(() {
this.participants = participants;
participantTracks = [...screenTracks, ...userMediaTracks];
});
}

Expand All @@ -139,18 +169,19 @@ class _RoomPageState extends State<RoomPage> {
body: Column(
children: [
Expanded(
child: participants.isNotEmpty
? ParticipantWidget.widgetFor(participants.first)
child: participantTracks.isNotEmpty
? ParticipantWidget.widgetFor(participantTracks.first)
: Container()),
SizedBox(
height: 100,
child: ListView.builder(
scrollDirection: Axis.horizontal,
itemCount: math.max(0, participants.length - 1),
itemCount: math.max(0, participantTracks.length - 1),
itemBuilder: (BuildContext context, int index) => SizedBox(
width: 100,
height: 100,
child: ParticipantWidget.widgetFor(participants[index + 1]),
child:
ParticipantWidget.widgetFor(participantTracks[index + 1]),
),
),
),
Expand Down
29 changes: 26 additions & 3 deletions example/lib/widgets/controls.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import 'package:eva_icons_flutter/eva_icons_flutter.dart';
import 'package:flutter/material.dart';
import 'package:flutter_background/flutter_background.dart';
import 'package:livekit_client/livekit_client.dart';
import 'package:flutter_webrtc/flutter_webrtc.dart';

import '../exts.dart';

Expand Down Expand Up @@ -86,9 +87,30 @@ class _ControlsWidgetState extends State<ControlsWidget> {
}

void _enableScreenShare() async {
await participant.setScreenShareEnabled(true);

if (Platform.isAndroid) {
if (WebRTC.platformIsDesktop) {
try {
final source = await showDialog<DesktopCapturerSource>(
context: context,
builder: (context) => ScreenSelectDialog(),
);
if (source == null) {
print('cancelled screenshare');
return;
}
print('DesktopCapturerSource: ${source.id}');
var track = await LocalVideoTrack.createScreenShareTrack(
ScreenShareCaptureOptions(
sourceId: source.id,
maxFrameRate: 15.0,
),
);
await participant.publishVideoTrack(track);
} catch (e) {
print('could not publish video: $e');
}
return;
}
if (WebRTC.platformIsAndroid) {
// Android specific
try {
// Required for android screenshare.
Expand All @@ -105,6 +127,7 @@ class _ControlsWidgetState extends State<ControlsWidget> {
print('could not publish video: $e');
}
}
await participant.setScreenShareEnabled(true);
}

void _disableScreenShare() async {
Expand Down
87 changes: 48 additions & 39 deletions example/lib/widgets/participant.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,25 @@ import 'participant_info.dart';

abstract class ParticipantWidget extends StatefulWidget {
// Convenience method to return relevant widget for participant
static ParticipantWidget widgetFor(Participant participant) {
if (participant is LocalParticipant) {
return LocalParticipantWidget(participant);
} else if (participant is RemoteParticipant) {
return RemoteParticipantWidget(participant);
static ParticipantWidget widgetFor(ParticipantTrack participantTrack) {
if (participantTrack.participant is LocalParticipant) {
return LocalParticipantWidget(
participantTrack.participant as LocalParticipant,
participantTrack.videoTrack,
participantTrack.isScreenShare);
} else if (participantTrack.participant is RemoteParticipant) {
return RemoteParticipantWidget(
participantTrack.participant as RemoteParticipant,
participantTrack.videoTrack,
participantTrack.isScreenShare);
}
throw UnimplementedError('Unknown participant type');
}

// Must be implemented by child class
abstract final Participant participant;
abstract final VideoTrack? videoTrack;
abstract final bool isScreenShare;
final VideoQuality quality;

const ParticipantWidget({
Expand All @@ -32,9 +40,15 @@ abstract class ParticipantWidget extends StatefulWidget {
class LocalParticipantWidget extends ParticipantWidget {
@override
final LocalParticipant participant;
@override
final VideoTrack? videoTrack;
@override
final bool isScreenShare;

const LocalParticipantWidget(
this.participant, {
this.participant,
this.videoTrack,
this.isScreenShare, {
Key? key,
}) : super(key: key);

Expand All @@ -45,9 +59,15 @@ class LocalParticipantWidget extends ParticipantWidget {
class RemoteParticipantWidget extends ParticipantWidget {
@override
final RemoteParticipant participant;
@override
final VideoTrack? videoTrack;
@override
final bool isScreenShare;

const RemoteParticipantWidget(
this.participant, {
this.participant,
this.videoTrack,
this.isScreenShare, {
Key? key,
}) : super(key: key);

Expand All @@ -60,7 +80,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
//
bool _visible = true;
VideoTrack? get activeVideoTrack;
TrackPublication? get firstVideoPublication;
TrackPublication? get videoPublication;
TrackPublication? get firstAudioPublication;

@override
Expand Down Expand Up @@ -89,12 +109,12 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
void _onParticipantChanged() => setState(() {});

// Widgets to show above the info bar
List<Widget> extraWidgets() => [];
List<Widget> extraWidgets(bool isScreenShare) => [];

@override
Widget build(BuildContext ctx) => Container(
foregroundDecoration: BoxDecoration(
border: widget.participant.isSpeaking
border: widget.participant.isSpeaking && !widget.isScreenShare
? Border.all(
width: 5,
color: LKColors.lkBlue,
Expand All @@ -112,7 +132,7 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
child: activeVideoTrack != null
? VideoTrackRenderer(
activeVideoTrack!,
fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover,
fit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain,
)
: const NoVideoWidget(),
),
Expand All @@ -124,14 +144,15 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
crossAxisAlignment: CrossAxisAlignment.stretch,
mainAxisSize: MainAxisSize.min,
children: [
...extraWidgets(),
...extraWidgets(widget.isScreenShare),
ParticipantInfoWidget(
title: widget.participant.name.isNotEmpty
? '${widget.participant.name} (${widget.participant.identity})'
: widget.participant.identity,
audioAvailable: firstAudioPublication?.muted == false &&
firstAudioPublication?.subscribed == true,
connectionQuality: widget.participant.connectionQuality,
isScreenShare: widget.isScreenShare,
),
],
),
Expand All @@ -144,60 +165,48 @@ abstract class _ParticipantWidgetState<T extends ParticipantWidget>
class _LocalParticipantWidgetState
extends _ParticipantWidgetState<LocalParticipantWidget> {
@override
LocalTrackPublication<LocalVideoTrack>? get firstVideoPublication =>
widget.participant.videoTracks.firstOrNull;
LocalTrackPublication<LocalVideoTrack>? get videoPublication =>
widget.participant.videoTracks
.where((element) => element.sid == widget.videoTrack?.sid)
.firstOrNull;

@override
LocalTrackPublication<LocalAudioTrack>? get firstAudioPublication =>
widget.participant.audioTracks.firstOrNull;

@override
VideoTrack? get activeVideoTrack {
if (firstVideoPublication?.subscribed == true &&
firstVideoPublication?.muted == false &&
_visible) {
return firstVideoPublication?.track;
}
return null;
}
VideoTrack? get activeVideoTrack => widget.videoTrack;
}

class _RemoteParticipantWidgetState
extends _ParticipantWidgetState<RemoteParticipantWidget> {
@override
RemoteTrackPublication<RemoteVideoTrack>? get firstVideoPublication =>
widget.participant.videoTracks.firstOrNull;
RemoteTrackPublication<RemoteVideoTrack>? get videoPublication =>
widget.participant.videoTracks
.where((element) => element.sid == widget.videoTrack?.sid)
.firstOrNull;

@override
RemoteTrackPublication<RemoteAudioTrack>? get firstAudioPublication =>
widget.participant.audioTracks.firstOrNull;

@override
VideoTrack? get activeVideoTrack {
for (final trackPublication in widget.participant.videoTracks) {
print(
'video track ${trackPublication.sid} subscribed ${trackPublication.subscribed} muted ${trackPublication.muted}');
if (trackPublication.subscribed && !trackPublication.muted && _visible) {
return trackPublication.track;
}
}
return null;
}
VideoTrack? get activeVideoTrack => widget.videoTrack;

@override
List<Widget> extraWidgets() => [
List<Widget> extraWidgets(bool isScreenShare) => [
Row(
mainAxisSize: MainAxisSize.max,
mainAxisAlignment: MainAxisAlignment.end,
children: [
// Menu for RemoteTrackPublication<RemoteVideoTrack>
if (firstVideoPublication != null)
if (videoPublication != null)
RemoteTrackPublicationMenuWidget(
pub: firstVideoPublication!,
icon: EvaIcons.video,
pub: videoPublication!,
icon: isScreenShare ? EvaIcons.monitor : EvaIcons.video,
),
// Menu for RemoteTrackPublication<RemoteAudioTrack>
if (firstAudioPublication != null)
if (firstAudioPublication != null && !isScreenShare)
RemoteTrackPublicationMenuWidget(
pub: firstAudioPublication!,
icon: EvaIcons.volumeUp,
Expand Down
Loading

0 comments on commit a554d92

Please sign in to comment.