From c734107a74888b7931984e9d6e1d0897d4501148 Mon Sep 17 00:00:00 2001 From: Daniel Abramov Date: Tue, 3 Oct 2023 18:20:09 +0200 Subject: [PATCH 1/2] example: replace logo track with screen-sharing --- examples/wgpu_room/src/app.rs | 4 +- examples/wgpu_room/src/logo_track.rs | 209 ------------------ examples/wgpu_room/src/main.rs | 2 +- examples/wgpu_room/src/moving-logo.png | Bin 7170 -> 0 bytes examples/wgpu_room/src/screensharing_track.rs | 149 +++++++++++++ examples/wgpu_room/src/service.rs | 16 +- 6 files changed, 160 insertions(+), 220 deletions(-) delete mode 100644 examples/wgpu_room/src/logo_track.rs delete mode 100644 examples/wgpu_room/src/moving-logo.png create mode 100644 examples/wgpu_room/src/screensharing_track.rs diff --git a/examples/wgpu_room/src/app.rs b/examples/wgpu_room/src/app.rs index 1d6ea0c1f..86c45ca71 100644 --- a/examples/wgpu_room/src/app.rs +++ b/examples/wgpu_room/src/app.rs @@ -153,8 +153,8 @@ impl LkApp { }); ui.menu_button("Publish", |ui| { - if ui.button("Logo").clicked() { - let _ = self.service.send(AsyncCmd::ToggleLogo); + if ui.button("Screen-sharing").clicked() { + let _ = self.service.send(AsyncCmd::ToggleScreensharing); } if ui.button("SineWave").clicked() { let _ = self.service.send(AsyncCmd::ToggleSine); diff --git a/examples/wgpu_room/src/logo_track.rs b/examples/wgpu_room/src/logo_track.rs deleted file mode 100644 index de93ec03f..000000000 --- a/examples/wgpu_room/src/logo_track.rs +++ /dev/null @@ -1,209 +0,0 @@ -use image::ImageFormat; -use image::RgbaImage; -use livekit::options::TrackPublishOptions; -use livekit::prelude::*; -use livekit::webrtc::video_source::RtcVideoSource; -use livekit::webrtc::video_source::VideoResolution; -use livekit::webrtc::{ - native::yuv_helper, - video_frame::native::I420BufferExt, - video_frame::{I420Buffer, VideoFrame, VideoRotation}, - video_source::native::NativeVideoSource, -}; -use parking_lot::Mutex; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::oneshot; -use tokio::task::JoinHandle; - -// The logo must not be bigger than the framebuffer -const PIXEL_SIZE: usize = 4; -const FRAME_RATE: u64 = 30; -const MOVE_SPEED: i32 = 16; -const FB_WIDTH: usize = 1280; -const FB_HEIGHT: usize = 720; - -#[derive(Clone)] -struct FrameData { - image: Arc, - framebuffer: Arc>>, - video_frame: Arc>>, - pos: (i32, i32), - direction: (i32, i32), -} - -struct TrackHandle { - close_tx: oneshot::Sender<()>, - track: LocalVideoTrack, - task: JoinHandle<()>, -} - -pub struct LogoTrack { - rtc_source: NativeVideoSource, - room: Arc, - handle: Option, -} - -impl LogoTrack { - pub fn new(room: Arc) -> Self { - Self { - rtc_source: NativeVideoSource::new(VideoResolution { - width: FB_WIDTH as u32, - height: FB_HEIGHT as u32, - }), - room, - handle: None, - } - } - - pub fn is_published(&self) -> bool { - self.handle.is_some() - } - - pub async fn publish(&mut self) -> Result<(), RoomError> { - self.unpublish().await?; - - let (close_tx, close_rx) = oneshot::channel(); - let track = LocalVideoTrack::create_video_track( - "livekit_logo", - RtcVideoSource::Native(self.rtc_source.clone()), - ); - - let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone())); - - self.room - .local_participant() - .publish_track( - LocalTrack::Video(track.clone()), - TrackPublishOptions { - source: TrackSource::Camera, - //simulcast: false, - ..Default::default() - }, - ) - .await?; - - let handle = TrackHandle { - close_tx, - task, - track, - }; - - self.handle = Some(handle); - Ok(()) - } - - pub async fn unpublish(&mut self) -> Result<(), RoomError> { - if let Some(handle) = self.handle.take() { - let _ = handle.close_tx.send(()); - let _ = handle.task.await; - - self.room - .local_participant() - .unpublish_track(&handle.track.sid()) - .await?; - } - Ok(()) - } - - async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeVideoSource) { - let mut interval = tokio::time::interval(Duration::from_millis(1000 / FRAME_RATE)); - - let image = tokio::task::spawn_blocking(|| { - image::load_from_memory_with_format(include_bytes!("moving-logo.png"), ImageFormat::Png) - .unwrap() - .to_rgba8() - }) - .await - .unwrap(); - - let mut data = FrameData { - image: Arc::new(image), - framebuffer: Arc::new(Mutex::new(vec![0u8; FB_WIDTH * FB_HEIGHT * 4])), - video_frame: Arc::new(Mutex::new(VideoFrame { - rotation: VideoRotation::VideoRotation0, - buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32), - timestamp_us: 0, - })), - pos: (0, 0), - direction: (1, 1), - }; - - loop { - tokio::select! { - _ = &mut close_rx => { - break; - } - _ = interval.tick() => {} - } - - data.pos.0 += data.direction.0 * MOVE_SPEED; - data.pos.1 += data.direction.1 * MOVE_SPEED; - - if data.pos.0 >= (FB_WIDTH - data.image.width() as usize) as i32 { - data.direction.0 = -1; - } else if data.pos.0 <= 0 { - data.direction.0 = 1; - } - - if data.pos.1 >= (FB_HEIGHT - data.image.height() as usize) as i32 { - data.direction.1 = -1; - } else if data.pos.1 <= 0 { - data.direction.1 = 1; - } - - tokio::task::spawn_blocking({ - let data = data.clone(); - let source = rtc_source.clone(); - move || { - let image = data.image.as_raw(); - let mut framebuffer = data.framebuffer.lock(); - let mut video_frame = data.video_frame.lock(); - let i420_buffer = &mut video_frame.buffer; - - let (stride_y, stride_u, stride_v) = i420_buffer.strides(); - let (data_y, data_u, data_v) = i420_buffer.data_mut(); - - framebuffer.fill(0); - for i in 0..data.image.height() as usize { - let x = data.pos.0 as usize; - let y = data.pos.1 as usize; - let frame_width = data.image.width() as usize; - let logo_stride = frame_width * PIXEL_SIZE; - let row_start = (x + ((i + y) * FB_WIDTH)) * PIXEL_SIZE; - let row_end = row_start + logo_stride; - - framebuffer[row_start..row_end].copy_from_slice( - &image[i * logo_stride..i * logo_stride + logo_stride], - ); - } - - yuv_helper::abgr_to_i420( - &framebuffer, - (FB_WIDTH * PIXEL_SIZE) as u32, - data_y, - stride_y, - data_u, - stride_u, - data_v, - stride_v, - FB_WIDTH as i32, - FB_HEIGHT as i32, - ); - - source.capture_frame(&*video_frame); - } - }) - .await - .unwrap(); - } - } -} - -impl Drop for LogoTrack { - fn drop(&mut self) { - if let Some(handle) = self.handle.take() { - let _ = handle.close_tx.send(()); - } - } -} diff --git a/examples/wgpu_room/src/main.rs b/examples/wgpu_room/src/main.rs index ffad45e99..bf53c5239 100644 --- a/examples/wgpu_room/src/main.rs +++ b/examples/wgpu_room/src/main.rs @@ -4,7 +4,7 @@ use std::thread; use std::time::Duration; mod app; -mod logo_track; +mod screensharing_track; mod service; mod sine_track; mod video_grid; diff --git a/examples/wgpu_room/src/moving-logo.png b/examples/wgpu_room/src/moving-logo.png deleted file mode 100644 index 5c47cc1edce2f10883583172d8e6a7df8d94beb6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7170 zcmcIpXHXMLv`(l>FCrjSq=cd{pow`(sVx!P}xMK%G>2PN8G`2Eboo=+aPa6K}7R^AQ+c`f0~~RvM%A(Zq`(Tu_JS zYFr0Q%?Ws#I;}|sNEP(Gn)81b|Evt)3okt_J=xn1xt5yw7cjh;s3VbybZrpVq3s1c z1lKG_(f_A@DDMB~^Zz#xi%gHy0Xp##HMT1d%Wiruh?w5w8#+KK+NiFt5gc#d5@3}o zduf29FXz`puTo0Np^QM)m zwn6}@;eL^^e%t|q50j0J+^c#&&q+IO=VRzLLP4%&rm~82zVjc>`SL9H5c4As^W60A%2jm@ihbilOc`0&M16v@ zB8|UiAAE&oUwZwurC&|KI#?)#)yr%2OBh*j#jA!UVD>id zHVVbua(0ER;agylDC96#1O_;BgM?QrroUi+xqc1rY{xl|{H&s;VHz|pen25R0Mgwl zAMsw2ty_mY>4VPAD@hLBfgN}0e+uE-^<1bQsVMZ^GnLcY9VzuN5G5Mx?$o~vvd+P2 z+gQXy9QC$qv%runu=3lH3$Kvpd<7Zc`G#7<(jU8K4ln%C6-th!LtT90Ny$Vu zh115+#%dy1?&g=20ohrWi0+p3+>F7MeX>>ipm!xY6&7wsD`Loos} z_+4>StJvkzO}i-FjN{AcWTTMywcx~nLtL;_bW?cREjOt4OZP*lEedgaL4(y_4VxI7 zWN7@{g?d@@o%u{V*_o`DPy-9ann#d)z0^b|!D={qwk}KMCf<^Mk=kEyH1OVmX0@*W zE$m$Sq=~1U!$hrQ5CA*6W^~jlH?f(8UtIwZ?xiK-gL<;Q_aX58{wv?ZH2?)#zL+=3 z7Q3hW&I&!Z7FfgYxY%frE8bLW)wINs@!Ar23}tS6lC=l6v-sb|iXq~|mt2O~k9o}83mZ4tK2>}_h{A}mm>f`Lyp8-C7;drB<4lhqw*;{rHcUSUUhg&k21tLe39i=?`>krqR7&(x2SA^V4>vxy4R(z@vwm&lPqKQ8OI?UDH0yv4obT0q`x})be zQ@)MI!=vsYQ=mw<2Rf&k2ID=TaI;Mw#7&NFp7OS*q?p0$KPw|btSD7)`a6!kND>Sb zg3yvVEd8?#%lG~=55ZqB8-=Djlc#H{e@+}u#2G<}H8lZfvTTCz%bVXrVLq%_@UBR6 zPg2zQH0QUGdE|#e3Pqha$bNE%oOC&~k2aCvOGufuasg{BXQh(V=j;ZxnsLa3q3wb< z8dH@D^SS~)%^2&$c{AX{w4(_JqHC2kXA427yea5c*J)we4OQ1}n>tdy5Nalt4G?YO z76_0Dz^3cqlt%sJ9W4U^V0@J98@;0N)-3=m1pgsRtP^9VrR7r zUi-{R(kl&BW%ZYrN%O6A7Sr2QkF?0mc`J(ra#W=n9tA7R86nA|`N_6~2aXD4lC{@W zvUXP>)(S3Tcv<_9mIDKFc&A!8J2{_lYEv*@8|RsZozFr5$fsn6d|Y@w-sZ7VPPk1mb`=unoHJr3=p!U5u^$-1DkC!NPFey1pMtevlL3{<;rd zQ+k1WFlI0or*Aojq=v2R+khC(ZDAMJGgO#d$1s)zoUv{lF=kK^2Z>AY(A&z%nJHlC zg3e`nW(JZ;3bLRH4t)-IX6SyJbT=qZ+dLWz!PzuHkj|HVcVuq5%vh4BnEFBw<@z0ZmD3)73ks!UIU(I2gh_eDn1d zGOgl3vMWR1Q>#KiHS;%B-u*o`P}ZcQXhA>n?__ricBFPpFOLmDJ~bl#o7|7Ou%09i zYFojo2>7e6&P(*89o*$=&ZwVI;C=X$>g|>5Tcq+OVckMZ+9w0v$D!9Zq~E%l(*yoa zj;MQMviNya#2T${QltjCySKE_SuQ!?9hQ-f`4dHC<9%fH z)JOo{y!%ezV?pPnH=ni&5PyoQ<4_M<-Hv^@>Vg^tP3GJ0u$^U%!0QD5@F%?XK<_r{ zlLbq7WyOVQBoym8TfuaA%bs_Stp5NNrRM^bov7nBnAjSZNT5g@jt>5;P%lBw#;nWH zR=^o#?boap%eTlXlKs~aot>R=23P#dgG6Jm4;A^C>$oP$vn8rcKVkqS9w_Bjzr+*U z(Vm%oH2G$5PwLg@_akh*o+V>Sz1A3L%*wuXO!zA!J1auLeOtYr-@d12c5{o{KQvl< zJfSGYc(^Bv+#(6s*2PUB{>T92sK6zUdW?q^;rZ1{ix0@ZE-oWJh)#TSYSXZOw++#Ob2-d6)YhEuO$b?kW zermj$=`os4D)u?2PqKGFyNvzK2pqwFlgj#NWR|iw$!EWy7kiLblWur0A#Y!xzsQqM zIp=0kDAMsu9b!BtLymXbUFuad%?K;<|J#gf+um$DD3zFS7S7eOP7^0&L3_Hi+`ZMZ zBDi%#S9K>hW^>S!e+yu$PW_5=*R3L9oGCzOjJG6Pn@KHmMaYb?`ojlHwR+04S6eO5 zg8&Y{YPZCL0n)TEsk7q|VXbXoy3H^ppp0?9YF~P7&3aQc?qGT``5nLSChc8^cr(Sn zCd_YgYxczNmL;5XyEs#1u?|;XLYJw*pCu&@IiHahW_ig6CmX`zofa;`oy4BBMEMhr zgl?>(rh4ww8edqM2HY(QbP{&FJ#R`LUbC4=embxd)z0Tt$UGcncU`Cdd9PfdYsS5y zlN%W#ucuEcapeK-jsg;!#h~r^%i#CdlB6O((A_81ZMA6s?K9%z`C>=a9k^^LI21tZ zh34p6if!XkPZJOsQtcU^RDa8=9;W)pn|9*rWpYEuZ(wA1B_~7LJ4uNq?>=N09=w*r zbf*6XI!$9UzMkJZUriXG-wy)U2~3JYYg4ACkNJp!(kWKkBK05H!(v)imH31TWWuy9 zax4#OIDh|WOZDf*+JKU`0$$$;P)s2wF=f99YI+~sO2k1H$E&h6J6!poEZP+Sifv(Ga@IfbQ ztf%@TeSW-hLmme3+fe$M416^g#PA^`hAWM=f)BjYDaQ;NDd}!^9CdZ40!T;tA*q61 z_*w$_Y>#^GaLC|oZ_ni4H&{Pts{IhC+NT?Z?!Ij-M=#QhekdBhENk#yjAwl>)>P4; znqSgSn{^4r4h++8aOe?(=f@odOQu`~NXV>WxU}LFKL&ptbXFEP!sJ$1Y$AUy{zEMH z#B{cLOB_fRDzb>Z2pp^4HCqz`bnNKmgB9~gzsunSg>TGTvsX+?zt2JCPlH^UyH`xx zq-%}SKqS#K))}m;=HerJ^XHp>)J0nBh~~bUxg~3`VS)NqZ*}tJxU(mxEgPP;WvMlM zpjrAUgj#$C`ZF`);Y(WPYpPKMRxAT7l+$71fI(IK2(l!LQGR>nQwp(RtMjS48rD{3 zn2ct)_HQdD5FX2Cy(p4`PTn>HE-G#i*9U7}kG5yqS<+1WRLkPHS5b*3Hh%7s@0^T(Rei?oMwVd1xU=D^U&gG^*@7&oK)Mk8m};-{8UUz`_@qO)6++rBOi&8q$KTCPD@b^CK~V>_%$fgDIo8zf|a*cU&4!tod-xP~-TL1iU=s1fYB*Gui) zAMZeY>6oQ4X>-Sl0_Wq8C6Kw^QM3J_x3&?U8_EJTnCO_!4)T~+y=6?Sz_)xJA_88D zp^NLYW78)V%ia(biI&5vn9uo(f8ZL?DGR@^l%fax@c|+tV-7nm|0=`-gJ#lYMo3J1 zFRO;m=-2*Y9Hmfv-1^9Vb6G#(h4sS|#YNrXR172GC;v+~-*G>SCkjkP@pu8G+D9mV zKZs9LMGM%Pn9EQV2BRX?rX4hQ)fF#-MlLiM{pAlk{qK02n{QsD16=r^#X$JE4zTra zARi#~cGC_eLiF{%-PJW34fwO$NLD*cAT_#j`V5#Vd$&94*WG5ZPgvTNOB9qQXDuwy z3*{V!*r`s7kvMYH5%$yYfY5@!=cXAcRq4)*onTc*{7+oB%~+Zdiu*AUQr6#@oa8FXRJImy0IxRB`iY>6WE z@~CkjK-__(VLtW8Z7u?}NiUx(xgHDO zCg>L0VQ8`cAwL#2;?S97u`hs}b9|;B$E(0`*B#Dswlw0PA*^BU`i6XjHYV+7M>m# zuz=Ur=~Tr+B-<^PwksN+`ks!*s2nk}h?RbIX^mN@I2&%?6We4vL=eu4Uq{jhcq;*b_B`)TULoyj*g z*9a_Z8s!n=Eh5bFwxCkJrMG{*<{WT9S#CHg;|6MeA+Rb9N(Gpn`Le*o7g=n&^ov;) zMkPV~SRbcp1k&KkQZ9#|e;~X%?&#f{hEcWHMPkW?;9+lvBePL8*D2T1r?xhp0#VQD z*A~snopLhKJG2OSP}a$jr~_I-F^W88=fVaF$|nyU3*` z9Y&pg_Wop&_6(#mM{Y+DL3EOWpC!jgX552=POd8nHW6w544eqr1=z3>tHVw4mfE3I zyY>5Ft)inzmlRmfY3+PQGW93=4+)HGZwc%TQGw=m2K$%2B?^wRvoDk{FWZo1a`@%= zg_#D1s){4!l@KBUA6Z!9p<$mTJHUw>E;8{<5{jBQ9d*0d?(I8SOLq}b4WT-#$+14r zTa>XuppAz2nc<(b!GgC1B+b}a&()uJLyHakpC7K|#sCPi#XNIQo39gBc9eIU!hgXS z>lz1rPiql%Y+-YSQJ2i*|9yRdONM$=uJ? zqTeWXGl+D3>P5ah%+FU<6m~RqoNvtb?M4bD7a#Tt87fX&Vc;2!yeLZX!8oe2Xam#^;OPFB8m6&;>72sd(+!XNdS|!KOEP78t0$|Lb*o?3aJ61s zlTT~B*}EbW=qHs}Zh}dMT^2)CCq8r<aZf+wRDB5Y4)SY$?0ZK>rW)-STs5CKT}LyUvRXyvA(%jzF`R936RHA6KrRk0#A`KmAE!M>Vrp7qgk_Iv z?0Wis+ueU6pg9Ivd+&Q|9Y+(K$#GLHPXNDrW1&F;#N>Cs!Dl1+?m-*--B`1Azn1I9 zK!vGs8pVB?4y`7Vyv#?gi+iX(5N_jk@1O6HG4jK%F{+0OOcSSUy&On5i$-29^>E&Y zNxEO_F;672#8xTntXx+eMzYku%lGn(M#kpRP_t^zc_dtxD%jIafAU3y$mL#nGeuSq zqsndhsp*$?xLdKVgIS;M9n7#ja!+7q)@3l)A?{D2P zPH#1|Wfnr~3D=iE$EoS0F=5Ht7rqTr;EO-b%~gr&OX8THRB)^_x^pL%2TF1`%hTzJ z6dsr;C3yduPqHgoby^bKe$Bj#C>HHW!y9+sIE_Yx89j46coMQ!i2<6=WN3_DWG!Gy zeWA?r3C`1VmZ4Ja&3x`5)K{5+*6ma6fAK@>e@e, + track: LocalVideoTrack, + task: JoinHandle<()>, +} + +pub struct ScreensharingTrack { + room: Arc, + handle: Option, +} + +impl ScreensharingTrack { + pub fn new(room: Arc) -> Self { + Self { room, handle: None } + } + + pub fn is_published(&self) -> bool { + self.handle.is_some() + } + + pub async fn publish(&mut self) -> Result<(), RoomError> { + self.unpublish().await?; + + let rtc_source = NativeVideoSource::new(VideoResolution { + width: WIDTH as u32, + height: HEIGHT as u32, + }); + + let track = LocalVideoTrack::create_video_track( + "native-screen-sharing", + RtcVideoSource::Native(rtc_source.clone()), + ); + + let (close_tx, close_rx) = oneshot::channel(); + let task = tokio::spawn(Self::track_task(close_rx, rtc_source)); + + self.room + .local_participant() + .publish_track( + LocalTrack::Video(track.clone()), + TrackPublishOptions { + source: TrackSource::Screenshare, + ..Default::default() + }, + ) + .await?; + + let handle = TrackHandle { + close_tx, + task, + track, + }; + + self.handle = Some(handle); + Ok(()) + } + + pub async fn unpublish(&mut self) -> Result<(), RoomError> { + if let Some(handle) = self.handle.take() { + let _ = handle.close_tx.send(()); + let _ = handle.task.await; + + self.room + .local_participant() + .unpublish_track(&handle.track.sid()) + .await?; + } + Ok(()) + } + + async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeVideoSource) { + const PIXEL_SIZE: usize = 4; + const FPS: usize = 15; + + let video_frame = Arc::new(Mutex::new(VideoFrame { + rotation: VideoRotation::VideoRotation0, + buffer: I420Buffer::new(WIDTH as u32, HEIGHT as u32), + timestamp_us: 0, + })); + + let mut interval = tokio::time::interval(Duration::from_millis(1000 / FPS as u64)); + loop { + tokio::select! { + _ = &mut close_rx => { + return; + } + _ = interval.tick() => {} + } + + tokio::task::spawn_blocking({ + // TODO: Use `spawn_blocking()` here to actually capture the native frame. + let image: RgbaImage = RgbaImage::new(WIDTH as u32, HEIGHT as u32); + + let (source, frame) = (rtc_source.clone(), video_frame.clone()); + move || { + let mut video_frame = frame.lock(); + let i420_buffer = &mut video_frame.buffer; + let (stride_y, stride_u, stride_v) = i420_buffer.strides(); + let (data_y, data_u, data_v) = i420_buffer.data_mut(); + + yuv_helper::abgr_to_i420( + &*image, + (WIDTH * PIXEL_SIZE) as u32, + data_y, + stride_y, + data_u, + stride_u, + data_v, + stride_v, + WIDTH as i32, + HEIGHT as i32, + ); + + source.capture_frame(&*video_frame); + } + }) + .await + .unwrap(); + } + } +} + +impl Drop for ScreensharingTrack { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + let _ = handle.close_tx.send(()); + } + } +} diff --git a/examples/wgpu_room/src/service.rs b/examples/wgpu_room/src/service.rs index e7bd6dfef..49929bda5 100644 --- a/examples/wgpu_room/src/service.rs +++ b/examples/wgpu_room/src/service.rs @@ -1,5 +1,5 @@ use crate::{ - logo_track::LogoTrack, + screensharing_track::ScreensharingTrack, sine_track::{SineParameters, SineTrack}, }; use livekit::{ @@ -24,7 +24,7 @@ pub enum AsyncCmd { SimulateScenario { scenario: SimulateScenario, }, - ToggleLogo, + ToggleScreensharing, ToggleSine, SubscribeTrack { publication: RemoteTrackPublication, @@ -97,7 +97,7 @@ impl LkService { async fn service_task(inner: Arc, mut cmd_rx: mpsc::UnboundedReceiver) { struct RunningState { room: Arc, - logo_track: LogoTrack, + screensharing_track: ScreensharingTrack, sine_track: SineTrack, } @@ -139,7 +139,7 @@ async fn service_task(inner: Arc, mut cmd_rx: mpsc::UnboundedRecei let new_room = Arc::new(new_room); running_state = Some(RunningState { room: new_room.clone(), - logo_track: LogoTrack::new(new_room.clone()), + screensharing_track: ScreensharingTrack::new(new_room.clone()), sine_track: SineTrack::new(new_room.clone(), SineParameters::default()), }); @@ -167,12 +167,12 @@ async fn service_task(inner: Arc, mut cmd_rx: mpsc::UnboundedRecei } } } - AsyncCmd::ToggleLogo => { + AsyncCmd::ToggleScreensharing => { if let Some(state) = running_state.as_mut() { - if state.logo_track.is_published() { - state.logo_track.unpublish().await.unwrap(); + if state.screensharing_track.is_published() { + state.screensharing_track.unpublish().await.unwrap(); } else { - state.logo_track.publish().await.unwrap(); + state.screensharing_track.publish().await.unwrap(); } } } From aa75db11c87c757887908962f069731950f646ce Mon Sep 17 00:00:00 2001 From: Daniel Abramov Date: Tue, 3 Oct 2023 19:45:27 +0200 Subject: [PATCH 2/2] example: use `screenshots` for screen-sharing --- examples/Cargo.lock | 330 ++++++++++++++++-- examples/wgpu_room/Cargo.toml | 25 +- examples/wgpu_room/src/screensharing_track.rs | 12 +- 3 files changed, 329 insertions(+), 38 deletions(-) diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 6da6503ea..2cebbd47d 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -503,8 +503,8 @@ dependencies = [ "block", "cocoa-foundation", "core-foundation", - "core-graphics", - "foreign-types", + "core-graphics 0.22.3", + "foreign-types 0.3.2", "libc", "objc", ] @@ -623,7 +623,20 @@ dependencies = [ "bitflags 1.3.2", "core-foundation", "core-graphics-types", - "foreign-types", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970a29baf4110c26fedbc7f82107d42c23f7e88e404c4577ed73fe99ff85a212" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types 0.5.0", "libc", ] @@ -776,6 +789,17 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +[[package]] +name = "dbus" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bb21987b9fb1613058ba3843121dd18b163b254d8a6e797e144cbac14d96d1b" +dependencies = [ + "libc", + "libdbus-sys", + "winapi", +] + [[package]] name = "deranged" version = "0.3.8" @@ -820,6 +844,20 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" +[[package]] +name = "display-info" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f9fb6a73233755f827129d80a80a6a16448122040537c881248a09d5c80ab6b" +dependencies = [ + "anyhow", + "core-graphics 0.23.1", + "fxhash", + "widestring", + "windows 0.48.0", + "xcb", +] + [[package]] name = "dlib" version = "0.5.2" @@ -1094,7 +1132,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", ] [[package]] @@ -1103,6 +1162,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -1211,6 +1276,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1305,7 +1379,7 @@ dependencies = [ "log", "thiserror", "winapi", - "windows", + "windows 0.44.0", ] [[package]] @@ -1608,6 +1682,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "io-lifetimes" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "ipnet" version = "2.8.0" @@ -1739,6 +1824,16 @@ version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +[[package]] +name = "libdbus-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06085512b750d640299b79be4bad3d2fa90a9c00b1fd9e1b46364f66f0485c72" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "libloading" version = "0.7.4" @@ -1759,6 +1854,22 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "libwayshot" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "896d0e594158b7f5188034836a6c4886492078352c39760786e54f1b796caaea" +dependencies = [ + "image", + "log", + "memmap2 0.7.1", + "nix 0.26.4", + "thiserror", + "wayland-client 0.30.2", + "wayland-protocols 0.30.1", + "wayland-protocols-wlr", +] + [[package]] name = "libwebrtc" version = "0.2.0" @@ -1904,6 +2015,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.6.5" @@ -1913,6 +2033,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.9.0" @@ -1931,7 +2060,7 @@ dependencies = [ "bitflags 1.3.2", "block", "core-graphics-types", - "foreign-types", + "foreign-types 0.3.2", "log", "objc", ] @@ -2081,6 +2210,19 @@ dependencies = [ "memoffset 0.6.5", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset 0.7.1", + "pin-utils", +] + [[package]] name = "nohash-hasher" version = "0.2.0" @@ -2267,7 +2409,7 @@ checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ "bitflags 2.4.0", "cfg-if", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -2616,6 +2758,15 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "quick-xml" +version = "0.28.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.33" @@ -2973,6 +3124,25 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" +[[package]] +name = "screenshots" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43888b127e056e3de0d4ffe995809e3f0226995ac5dc2daffba0a116782fabb1" +dependencies = [ + "anyhow", + "core-graphics 0.22.3", + "dbus", + "display-info", + "fxhash", + "image", + "libwayshot", + "percent-encoding", + "widestring", + "windows 0.51.1", + "xcb", +] + [[package]] name = "sct" version = "0.7.0" @@ -2991,7 +3161,7 @@ checksum = "cda4e97be1fd174ccc2aae81c8b694e803fa99b34e8fd0f057a9d70698e3ed09" dependencies = [ "ab_glyph", "log", - "memmap2", + "memmap2 0.5.10", "smithay-client-toolkit", "tiny-skia", ] @@ -3149,12 +3319,12 @@ dependencies = [ "dlib", "lazy_static", "log", - "memmap2", + "memmap2 0.5.10", "nix 0.24.3", "pkg-config", - "wayland-client", + "wayland-client 0.29.5", "wayland-cursor", - "wayland-protocols", + "wayland-protocols 0.29.5", ] [[package]] @@ -3164,7 +3334,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a345c870a1fae0b1b779085e81b51e614767c239e93503588e54c5b17f4b0e8" dependencies = [ "smithay-client-toolkit", - "wayland-client", + "wayland-client 0.29.5", ] [[package]] @@ -3843,6 +4013,21 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +[[package]] +name = "wayland-backend" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b48e27457e8da3b2260ac60d0a94512f5cba36448679f3747c0865b7893ed8" +dependencies = [ + "cc", + "downcast-rs", + "io-lifetimes", + "nix 0.26.4", + "scoped-tls", + "smallvec", + "wayland-sys 0.30.1", +] + [[package]] name = "wayland-client" version = "0.29.5" @@ -3855,8 +4040,20 @@ dependencies = [ "nix 0.24.3", "scoped-tls", "wayland-commons", - "wayland-scanner", - "wayland-sys", + "wayland-scanner 0.29.5", + "wayland-sys 0.29.5", +] + +[[package]] +name = "wayland-client" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489c9654770f674fc7e266b3c579f4053d7551df0ceb392f153adb1f9ed06ac8" +dependencies = [ + "bitflags 1.3.2", + "nix 0.26.4", + "wayland-backend", + "wayland-scanner 0.30.1", ] [[package]] @@ -3868,7 +4065,7 @@ dependencies = [ "nix 0.24.3", "once_cell", "smallvec", - "wayland-sys", + "wayland-sys 0.29.5", ] [[package]] @@ -3878,7 +4075,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6865c6b66f13d6257bef1cd40cbfe8ef2f150fb8ebbdb1e8e873455931377661" dependencies = [ "nix 0.24.3", - "wayland-client", + "wayland-client 0.29.5", "xcursor", ] @@ -3889,9 +4086,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b950621f9354b322ee817a23474e479b34be96c2e909c14f7bc0100e9a970bc6" dependencies = [ "bitflags 1.3.2", - "wayland-client", + "wayland-client 0.29.5", "wayland-commons", - "wayland-scanner", + "wayland-scanner 0.29.5", +] + +[[package]] +name = "wayland-protocols" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b28101e5ca94f70461a6c2d610f76d85ad223d042dd76585ab23d3422dd9b4d" +dependencies = [ + "bitflags 1.3.2", + "wayland-backend", + "wayland-client 0.30.2", + "wayland-scanner 0.30.1", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce991093320e4a6a525876e6b629ab24da25f9baef0c2e0080ad173ec89588a" +dependencies = [ + "bitflags 1.3.2", + "wayland-backend", + "wayland-client 0.30.2", + "wayland-protocols 0.30.1", + "wayland-scanner 0.30.1", ] [[package]] @@ -3905,6 +4127,17 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "wayland-scanner" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b873b257fbc32ec909c0eb80dea312076a67014e65e245f5eb69a6b8ab330e" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + [[package]] name = "wayland-sys" version = "0.29.5" @@ -3916,6 +4149,17 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "wayland-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b2a02ac608e07132978689a6f9bf4214949c85998c247abadd4f4129b1aa06" +dependencies = [ + "dlib", + "log", + "pkg-config", +] + [[package]] name = "web-sys" version = "0.3.64" @@ -4058,7 +4302,7 @@ dependencies = [ "block", "core-graphics-types", "d3d12", - "foreign-types", + "foreign-types 0.3.2", "glow", "gpu-alloc", "gpu-allocator", @@ -4111,6 +4355,7 @@ dependencies = [ "livekit", "log", "parking_lot", + "screenshots", "serde", "tokio", "wgpu", @@ -4184,6 +4429,34 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +dependencies = [ + "windows-core", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-core" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +dependencies = [ + "windows-targets 0.48.5", +] + [[package]] name = "windows-sys" version = "0.45.0" @@ -4326,7 +4599,7 @@ dependencies = [ "bitflags 1.3.2", "cfg_aliases", "core-foundation", - "core-graphics", + "core-graphics 0.22.3", "dispatch", "instant", "libc", @@ -4342,10 +4615,10 @@ dependencies = [ "sctk-adwaita", "smithay-client-toolkit", "wasm-bindgen", - "wayland-client", + "wayland-client 0.29.5", "wayland-commons", - "wayland-protocols", - "wayland-scanner", + "wayland-protocols 0.29.5", + "wayland-scanner 0.29.5", "web-sys", "windows-sys 0.45.0", "x11-dl", @@ -4403,6 +4676,17 @@ dependencies = [ "nix 0.24.3", ] +[[package]] +name = "xcb" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb3acf6b0945550d37d3a683b8f7de9d9f66b2c14dc390313b34d7ac6f1b4089" +dependencies = [ + "bitflags 1.3.2", + "libc", + "quick-xml", +] + [[package]] name = "xcursor" version = "0.3.4" diff --git a/examples/wgpu_room/Cargo.toml b/examples/wgpu_room/Cargo.toml index f0d3b8c51..9d5fb6007 100644 --- a/examples/wgpu_room/Cargo.toml +++ b/examples/wgpu_room/Cargo.toml @@ -1,24 +1,25 @@ [package] +edition = "2021" name = "wgpu_room" version = "0.1.1" -edition = "2021" [features] default = [] tracing = ["console-subscriber", "tokio/tracing"] [dependencies] -tokio = { version = "1", features = ["full", "parking_lot"] } -livekit = { path = "../../livekit", version = "0.2.0", features = ["native-tls"] } -futures = "0.3" -winit = "0.28" -parking_lot = { version = "0.12.1", features = ["deadlock_detection"] } -image = "0.24" -wgpu = "0.16" +console-subscriber = {version = "0.1.10", features = ["parking_lot"], optional = true} +eframe = {version = "0.22.0", default-features = false, features = ["default_fonts", "wgpu", "persistence"]} egui = "0.22.0" egui-wgpu = "0.22.0" -eframe = { version = "0.22.0", default-features = false, features = ["default_fonts", "wgpu", "persistence"] } -serde = { version = "1", features = ["derive"] } -log = "0.4" env_logger = "0.10.0" -console-subscriber = { version = "0.1.10", features = ["parking_lot"], optional = true } \ No newline at end of file +futures = "0.3" +image = "0.24" +livekit = {path = "../../livekit", version = "0.2.0", features = ["native-tls"]} +log = "0.4" +parking_lot = {version = "0.12.1", features = ["deadlock_detection"]} +screenshots = "0.8.4" +serde = {version = "1", features = ["derive"]} +tokio = {version = "1", features = ["full", "parking_lot"]} +wgpu = "0.16" +winit = "0.28" diff --git a/examples/wgpu_room/src/screensharing_track.rs b/examples/wgpu_room/src/screensharing_track.rs index 0c26295e1..0d03f8abf 100644 --- a/examples/wgpu_room/src/screensharing_track.rs +++ b/examples/wgpu_room/src/screensharing_track.rs @@ -1,4 +1,7 @@ -use image::RgbaImage; +use image::{ + imageops::{resize, FilterType}, + RgbaImage, +}; use livekit::options::TrackPublishOptions; use livekit::prelude::*; use livekit::webrtc::video_source::RtcVideoSource; @@ -98,6 +101,9 @@ impl ScreensharingTrack { timestamp_us: 0, })); + let screens = screenshots::Screen::all().expect("Could not get access to the screens"); + let primary_screen = screens.first().expect("No screens found"); + let mut interval = tokio::time::interval(Duration::from_millis(1000 / FPS as u64)); loop { tokio::select! { @@ -108,8 +114,8 @@ impl ScreensharingTrack { } tokio::task::spawn_blocking({ - // TODO: Use `spawn_blocking()` here to actually capture the native frame. - let image: RgbaImage = RgbaImage::new(WIDTH as u32, HEIGHT as u32); + let captured = primary_screen.capture().expect("Could not capture screen"); + let image = resize(&captured, WIDTH as u32, HEIGHT as u32, FilterType::Lanczos3); let (source, frame) = (rtc_source.clone(), video_frame.clone()); move || {