diff --git a/Cargo.lock b/Cargo.lock index c9b86816..87d41ad9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" dependencies = [ "memchr", ] @@ -45,19 +45,19 @@ checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -68,9 +68,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", @@ -113,9 +113,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -152,9 +152,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "block-buffer" @@ -206,11 +206,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -373,9 +374,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.102" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f68e12e817cb19eaab81aaec582b4052d07debd3c3c6b083b9d361db47c7dc9d" +checksum = "28403c86fc49e3401fdf45499ba37fad6493d9329449d6449d7f0e10f4654d28" dependencies = [ "cc", "cxxbridge-flags", @@ -385,9 +386,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.102" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e789217e4ab7cf8cc9ce82253180a9fe331f35f5d339f0ccfe0270b39433f397" +checksum = "78da94fef01786dc3e0c76eafcd187abcaa9972c78e05ff4041e24fdf059c285" dependencies = [ "cc", "codespan-reporting", @@ -395,31 +396,31 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "cxxbridge-flags" -version = "1.0.102" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78a19f4c80fd9ab6c882286fa865e92e07688f4387370a209508014ead8751d0" +checksum = "e2a6f5e1dfb4b34292ad4ea1facbfdaa1824705b231610087b00b17008641809" [[package]] name = "cxxbridge-macro" -version = "1.0.102" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcfa71f66c8563c4fa9dd2bb68368d50267856f831ac5d85367e0805f9606c" +checksum = "50c49547d73ba8dcfd4ad7325d64c6d5391ff4224d498fc39a6f3f49825a530d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "dashmap" -version = "5.5.0" +version = "5.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" +checksum = "edd72493923899c6f10c641bdbdeddc7183d6396641d99c1a0d1597f37f92e28" dependencies = [ "cfg-if", "hashbrown 0.14.0", @@ -436,9 +437,9 @@ checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "deranged" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" [[package]] name = "digest" @@ -465,9 +466,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] @@ -485,6 +486,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.2" @@ -520,9 +527,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -624,7 +631,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -680,9 +687,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "glob" @@ -692,9 +699,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -702,7 +709,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -785,9 +792,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -812,7 +819,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -878,6 +885,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "inout" version = "0.1.3" @@ -1040,7 +1057,7 @@ dependencies = [ [[package]] name = "livekit-ffi" -version = "0.2.1" +version = "0.2.2" dependencies = [ "console-subscriber", "dashmap", @@ -1108,9 +1125,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "matchers" @@ -1123,9 +1140,9 @@ dependencies = [ [[package]] name = "matchit" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67827e6ea8ee8a7c4a72227ef4fc08957040acffdb5f122733b24fa12daff41b" +checksum = "ed1202b2a6f884ae56f04cff409ab315c5ce26b5e58d7412e484f01fd52f52ef" [[package]] name = "memchr" @@ -1220,9 +1237,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "memchr", ] @@ -1235,9 +1252,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if", @@ -1256,7 +1273,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -1267,18 +1284,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.26.0+1.1.1u" +version = "111.27.0+1.1.1v" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc62c9f12b22b8f5208c23a7200a442b2e5999f8bdf80233852122b5a4f6f37" +checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac" dependencies = [ "cc", "libc", @@ -1310,7 +1327,7 @@ dependencies = [ "redox_syscall 0.3.5", "smallvec", "thread-id", - "windows-targets 0.48.1", + "windows-targets 0.48.5", ] [[package]] @@ -1381,39 +1398,39 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.0.0", ] [[package]] name = "pin-project" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -1508,9 +1525,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -1565,13 +1582,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.4", + "regex-automata 0.3.6", "regex-syntax 0.7.4", ] @@ -1586,9 +1603,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", @@ -1609,9 +1626,9 @@ checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" dependencies = [ "base64 0.21.2", "bytes", @@ -1646,7 +1663,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", "winreg", ] @@ -1672,11 +1689,11 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.4" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", @@ -1685,13 +1702,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", - "rustls-webpki 0.101.2", + "rustls-webpki 0.101.4", "sct", ] @@ -1718,9 +1735,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ "ring", "untrusted", @@ -1728,9 +1745,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.2" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -1819,29 +1836,29 @@ checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.180" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" +checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.180" +version = "1.0.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" +checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" dependencies = [ "itoa", "ryu", @@ -1902,9 +1919,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -1925,6 +1942,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -1950,9 +1977,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a" dependencies = [ "proc-macro2", "quote", @@ -1967,9 +1994,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "tempfile" -version = "3.7.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", @@ -1989,29 +2016,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] name = "thread-id" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee93aa2b8331c0fec9091548843f2c90019571814057da3b783f9de09349d73" +checksum = "79474f573561cdc4871a0de34a51c92f7f5a56039113fbb5b9c9f96bdb756669" dependencies = [ "libc", "redox_syscall 0.2.16", @@ -2030,9 +2057,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.24" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b" +checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" dependencies = [ "deranged", "serde", @@ -2062,11 +2089,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.29.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", "backtrace", "bytes", "libc", @@ -2075,7 +2101,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.3", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -2099,7 +2125,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -2201,7 +2227,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -2245,7 +2271,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", ] [[package]] @@ -2421,7 +2447,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -2455,7 +2481,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2476,33 +2502,20 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "webpki", + "rustls-webpki 0.100.2", ] [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.1", -] +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "webrtc-sys" @@ -2586,7 +2599,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.1", + "windows-targets 0.48.5", ] [[package]] @@ -2606,17 +2619,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -2627,9 +2640,9 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" @@ -2639,9 +2652,9 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" @@ -2651,9 +2664,9 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" @@ -2663,9 +2676,9 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" @@ -2675,9 +2688,9 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" @@ -2687,9 +2700,9 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" @@ -2699,17 +2712,18 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 6a60ecb0..cfcedfdd 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -1863,9 +1863,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "malloc_buf" @@ -2472,6 +2472,17 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "play_from_disk" +version = "0.1.0" +dependencies = [ + "env_logger", + "livekit", + "log", + "thiserror", + "tokio", +] + [[package]] name = "png" version = "0.17.9" @@ -3258,18 +3269,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 0c4fe02d..2adfd7d8 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -3,6 +3,7 @@ members = [ "basic_room", "mobile", "save_to_disk", + "play_from_disk", "wgpu_room", "webhooks", ] diff --git a/examples/play_from_disk/.gitattributes b/examples/play_from_disk/.gitattributes new file mode 100644 index 00000000..6a53486f --- /dev/null +++ b/examples/play_from_disk/.gitattributes @@ -0,0 +1 @@ +*.wav filter=lfs diff=lfs merge=lfs -text \ No newline at end of file diff --git a/examples/play_from_disk/Cargo.toml b/examples/play_from_disk/Cargo.toml new file mode 100644 index 00000000..fda15b43 --- /dev/null +++ b/examples/play_from_disk/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "play_from_disk" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tokio = { version = "1", features = ["full"] } +livekit = { path = "../../livekit", version = "0.2.0" } +thiserror = "1.0.47" +log = "0.4.20" +env_logger = "0.10.0" diff --git a/examples/play_from_disk/change-sophie.wav b/examples/play_from_disk/change-sophie.wav new file mode 100644 index 00000000..80a9c6af --- /dev/null +++ b/examples/play_from_disk/change-sophie.wav @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16a978d4be2506cca5e25c5b7aa5b0e53451049d979684d9168c1a81beb4babd +size 570264 diff --git a/examples/play_from_disk/src/main.rs b/examples/play_from_disk/src/main.rs new file mode 100644 index 00000000..b96d4e5c --- /dev/null +++ b/examples/play_from_disk/src/main.rs @@ -0,0 +1,191 @@ +use livekit::{ + options::TrackPublishOptions, + track::{LocalAudioTrack, LocalTrack, TrackSource}, + webrtc::{ + audio_source::native::NativeAudioSource, + prelude::{AudioFrame, AudioSourceOptions, RtcAudioSource}, + }, + Room, RoomOptions, +}; +use std::{env, mem::size_of, sync::Arc, time::Duration}; +use std::{error::Error, io}; +use thiserror::Error; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader}; + +#[derive(Debug, Error)] +pub enum WavError { + #[error("Invalid header: {0}")] + InvalidHeader(&'static str), + #[error("IO error: {0}")] + Io(#[from] io::Error), +} + +pub struct WavReader { + reader: R, +} + +#[allow(dead_code)] +#[derive(Debug)] +pub struct WavHeader { + file_size: u32, + data_size: u32, + format: String, + format_length: u32, + format_type: u16, + num_channels: u16, + sample_rate: u32, + byte_rate: u32, + block_align: u16, + bits_per_sample: u16, +} + +impl WavReader { + pub fn new(reader: R) -> Self { + Self { reader } + } + + pub async fn read_header(&mut self) -> Result { + let mut header = [0u8; 4]; + let mut format = [0u8; 4]; + let mut chunk_marker = [0u8; 4]; + let mut data_chunk = [0u8; 4]; + + self.reader.read_exact(&mut header).await?; + + if &header != b"RIFF" { + return Err(WavError::InvalidHeader("Invalid RIFF header")); + } + + let file_size = self.reader.read_u32_le().await?; + self.reader.read_exact(&mut format).await?; + + if &format != b"WAVE" { + return Err(WavError::InvalidHeader("Invalid WAVE header")); + } + + self.reader.read_exact(&mut chunk_marker).await?; + + if &chunk_marker != b"fmt " { + return Err(WavError::InvalidHeader("Invalid fmt chunk")); + } + + let format_length = self.reader.read_u32_le().await?; + let format_type = self.reader.read_u16_le().await?; + let num_channels = self.reader.read_u16_le().await?; + let sample_rate = self.reader.read_u32_le().await?; + let byte_rate = self.reader.read_u32_le().await?; + let block_align = self.reader.read_u16_le().await?; + let bits_per_sample = self.reader.read_u16_le().await?; + self.reader.read_exact(&mut data_chunk).await?; + let data_size = self.reader.read_u32_le().await?; + + if &data_chunk != b"data" { + return Err(WavError::InvalidHeader("Invalid data chunk")); + } + + Ok(WavHeader { + file_size, + data_size, + format: String::from_utf8_lossy(&format).to_string(), + format_length, + format_type, + num_channels, + sample_rate, + byte_rate, + block_align, + bits_per_sample, + }) + } + + pub async fn read_i16(&mut self) -> Result { + let i = self.reader.read_i16_le().await?; + Ok(i) + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::init(); + + let url = env::var("LIVEKIT_URL").expect("LIVEKIT_URL is not set"); + let token = env::var("LIVEKIT_TOKEN").expect("LIVEKIT_TOKEN is not set"); + + let file = tokio::fs::File::open("change-sophie.wav").await?; + let mut reader = WavReader::new(BufReader::new(file)); + let header = reader.read_header().await?; + log::debug!("{:?}", header); + + if header.bits_per_sample != 16 { + return Err("only 16-bit samples supported for this demo".into()); + } + + let (room, mut rx) = Room::connect(&url, &token, RoomOptions::default()) + .await + .unwrap(); + let room = Arc::new(room); + log::info!("Connected to room: {} - {}", room.name(), room.sid()); + + let source = NativeAudioSource::new( + AudioSourceOptions::default(), + header.sample_rate, + header.num_channels as u32, + ); + + let track = LocalAudioTrack::create_audio_track("file", RtcAudioSource::Native(source.clone())); + + room.local_participant() + .publish_track( + LocalTrack::Audio(track), + TrackPublishOptions { + source: TrackSource::Microphone, + ..Default::default() + }, + ) + .await?; + + // Play the wav file and disconnect + tokio::spawn({ + let room = room.clone(); + async move { + const FRAME_DURATION: Duration = Duration::from_millis(1000); // Write 1s of audio at a time + + let max_samples = header.data_size as usize / size_of::(); + let ms = FRAME_DURATION.as_millis() as u32; + let num_samples = (header.sample_rate / 1000 * ms) as usize; + + log::info!("sample_rate: {}", header.sample_rate); + log::info!("num_channels: {}", header.num_channels); + log::info!("max samples: {}", max_samples); + log::info!("chunk size: {}ms - {} samples", ms, num_samples); + + let mut written_samples = 0; + while written_samples < max_samples { + let available_samples = max_samples - written_samples; + let frame_size = num_samples.min(available_samples); + + let mut audio_frame = AudioFrame { + data: vec![0i16; frame_size].into(), + num_channels: header.num_channels as u32, + sample_rate: header.sample_rate, + samples_per_channel: (frame_size / header.num_channels as usize) as u32, + }; + + for i in 0..frame_size { + let sample = reader.read_i16().await.unwrap(); + audio_frame.data.to_mut()[i] = sample; + } + + source.capture_frame(&audio_frame).await.unwrap(); + written_samples += frame_size; + } + + room.close().await.unwrap(); + } + }); + + while let Some(msg) = rx.recv().await { + log::info!("Event: {:?}", msg); + } + + Ok(()) +} diff --git a/examples/wgpu_room/src/logo_track.rs b/examples/wgpu_room/src/logo_track.rs index 3ea8540e..de93ec03 100644 --- a/examples/wgpu_room/src/logo_track.rs +++ b/examples/wgpu_room/src/logo_track.rs @@ -28,7 +28,7 @@ struct FrameData { image: Arc, framebuffer: Arc>>, video_frame: Arc>>, - pos: (u32, u32), + pos: (i32, i32), direction: (i32, i32), } @@ -119,7 +119,7 @@ impl LogoTrack { let mut data = FrameData { image: Arc::new(image), - framebuffer: Arc::new(Mutex::new(vec![0u8; (FB_WIDTH * FB_HEIGHT * 4) as usize])), + framebuffer: Arc::new(Mutex::new(vec![0u8; FB_WIDTH * FB_HEIGHT * 4])), video_frame: Arc::new(Mutex::new(VideoFrame { rotation: VideoRotation::VideoRotation0, buffer: I420Buffer::new(FB_WIDTH as u32, FB_HEIGHT as u32), @@ -137,16 +137,16 @@ impl LogoTrack { _ = interval.tick() => {} } - data.pos.0 = (data.pos.0 as i32 + data.direction.0 * MOVE_SPEED) as u32; - data.pos.1 = (data.pos.1 as i32 + data.direction.1 * MOVE_SPEED) as u32; + data.pos.0 += data.direction.0 * MOVE_SPEED; + data.pos.1 += data.direction.1 * MOVE_SPEED; - if data.pos.0 >= (FB_WIDTH - data.image.width() as usize) as u32 { + if data.pos.0 >= (FB_WIDTH - data.image.width() as usize) as i32 { data.direction.0 = -1; } else if data.pos.0 <= 0 { data.direction.0 = 1; } - if data.pos.1 >= (FB_HEIGHT - data.image.height() as usize) as u32 { + if data.pos.1 >= (FB_HEIGHT - data.image.height() as usize) as i32 { data.direction.1 = -1; } else if data.pos.1 <= 0 { data.direction.1 = 1; @@ -189,8 +189,7 @@ impl LogoTrack { stride_v, FB_WIDTH as i32, FB_HEIGHT as i32, - ) - .unwrap(); + ); source.capture_frame(&*video_frame); } diff --git a/examples/wgpu_room/src/service.rs b/examples/wgpu_room/src/service.rs index 74f798f2..8903739f 100644 --- a/examples/wgpu_room/src/service.rs +++ b/examples/wgpu_room/src/service.rs @@ -1,4 +1,7 @@ -use crate::{logo_track::LogoTrack, sine_track::SineTrack}; +use crate::{ + logo_track::LogoTrack, + sine_track::{SineParameters, SineTrack}, +}; use livekit::{prelude::*, SimulateScenario}; use parking_lot::Mutex; use std::sync::Arc; @@ -120,7 +123,7 @@ async fn service_task(inner: Arc, mut cmd_rx: mpsc::UnboundedRecei running_state = Some(RunningState { room: new_room.clone(), logo_track: LogoTrack::new(new_room.clone()), - sine_track: SineTrack::new(new_room.clone()), + sine_track: SineTrack::new(new_room.clone(), SineParameters::default()), }); // Allow direct access to the room from the UI (Used for sync access) diff --git a/examples/wgpu_room/src/sine_track.rs b/examples/wgpu_room/src/sine_track.rs index b88f77df..c21a9aaf 100644 --- a/examples/wgpu_room/src/sine_track.rs +++ b/examples/wgpu_room/src/sine_track.rs @@ -1,27 +1,27 @@ use livekit::options::TrackPublishOptions; use livekit::webrtc::audio_frame::AudioFrame; use livekit::webrtc::audio_source::RtcAudioSource; +use livekit::webrtc::prelude::AudioSourceOptions; use livekit::{prelude::*, webrtc::audio_source::native::NativeAudioSource}; use std::sync::Arc; -use std::time::Duration; use tokio::sync::oneshot; use tokio::task::JoinHandle; #[derive(Clone)] -struct FrameData { +pub struct SineParameters { pub sample_rate: u32, pub freq: f64, pub amplitude: f64, - pub phase: u64, + pub num_channels: u32, } -impl Default for FrameData { +impl Default for SineParameters { fn default() -> Self { Self { sample_rate: 48000, freq: 440.0, amplitude: 1.0, - phase: 0, + num_channels: 2, } } } @@ -34,14 +34,20 @@ struct TrackHandle { pub struct SineTrack { rtc_source: NativeAudioSource, + params: SineParameters, room: Arc, handle: Option, } impl SineTrack { - pub fn new(room: Arc) -> Self { + pub fn new(room: Arc, params: SineParameters) -> Self { Self { - rtc_source: NativeAudioSource::default(), + rtc_source: NativeAudioSource::new( + AudioSourceOptions::default(), + params.sample_rate, + params.num_channels, + ), + params, room, handle: None, } @@ -58,7 +64,11 @@ impl SineTrack { RtcAudioSource::Native(self.rtc_source.clone()), ); - let task = tokio::spawn(Self::track_task(close_rx, self.rtc_source.clone())); + let task = tokio::spawn(Self::track_task( + close_rx, + self.rtc_source.clone(), + self.params.clone(), + )); self.room .local_participant() @@ -94,50 +104,46 @@ impl SineTrack { Ok(()) } - async fn track_task(mut close_rx: oneshot::Receiver<()>, rtc_source: NativeAudioSource) { - let mut data = FrameData::default(); - let mut interval = tokio::time::interval(Duration::from_millis(10)); - let mut samples_10ms = Vec::::new(); - + async fn track_task( + mut close_rx: oneshot::Receiver<()>, + rtc_source: NativeAudioSource, + params: SineParameters, + ) { + let num_channels = params.num_channels as usize; + let samples_count = (params.sample_rate / 100) as usize * num_channels; + let mut samples_10ms = vec![0; samples_count]; + let mut phase = 0; loop { - tokio::select! { - _ = &mut close_rx => { - break; - } - _ = interval.tick() => {} - } - - const NUM_CHANNELS: usize = 2; - - let samples_count_10ms = (data.sample_rate / 100) as usize * NUM_CHANNELS; - - if samples_10ms.capacity() != samples_count_10ms { - samples_10ms.resize(samples_count_10ms, 0i16); + if close_rx.try_recv().is_ok() { + break; } - for i in (0..samples_count_10ms).step_by(NUM_CHANNELS) { - let val = data.amplitude + for i in (0..samples_count).step_by(num_channels) { + let val = params.amplitude * f64::sin( std::f64::consts::PI * 2.0 - * data.freq - * (data.phase as f64 / data.sample_rate as f64), + * params.freq + * (phase as f64 / params.sample_rate as f64), ); - data.phase += 1; + phase += 1; - for c in 0..NUM_CHANNELS { + for c in 0..num_channels { // WebRTC uses 16-bit signed PCM samples_10ms[i + c] = (val * 32768.0) as i16; } } - rtc_source.capture_frame(&AudioFrame { - data: samples_10ms.clone(), - sample_rate: data.sample_rate as u32, - num_channels: NUM_CHANNELS as u32, - samples_per_channel: samples_count_10ms as u32, - }); + rtc_source + .capture_frame(&AudioFrame { + data: samples_10ms.as_slice().into(), + sample_rate: params.sample_rate, + num_channels: params.num_channels, + samples_per_channel: samples_count as u32 / params.num_channels, + }) + .await + .unwrap(); } } } diff --git a/examples/wgpu_room/src/video_renderer.rs b/examples/wgpu_room/src/video_renderer.rs index ce3d82c6..0b584d14 100644 --- a/examples/wgpu_room/src/video_renderer.rs +++ b/examples/wgpu_room/src/video_renderer.rs @@ -72,8 +72,7 @@ impl VideoRenderer { rgba_stride, buffer.width() as i32, buffer.height() as i32, - ) - .unwrap(); + ); internal.render_state.queue.write_texture( wgpu::ImageCopyTexture { diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto index b486edf7..7628533e 100644 --- a/livekit-ffi/protocol/audio_frame.proto +++ b/livekit-ffi/protocol/audio_frame.proto @@ -27,7 +27,7 @@ message AllocAudioBufferRequest { uint32 num_channels = 2; uint32 samples_per_channel = 3; } -message AllocAudioBufferResponse { AudioFrameBufferInfo buffer = 1; } +message AllocAudioBufferResponse { OwnedAudioFrameBuffer buffer = 1; } // Create a new AudioStream // AudioStream is used to receive audio frames from a track @@ -35,38 +35,47 @@ message NewAudioStreamRequest { uint64 track_handle = 1; AudioStreamType type = 2; } -message NewAudioStreamResponse { AudioStreamInfo stream = 1; } +message NewAudioStreamResponse { OwnedAudioStream stream = 1; } // Create a new AudioSource message NewAudioSourceRequest { AudioSourceType type = 1; optional AudioSourceOptions options = 2; + uint32 sample_rate = 3; + uint32 num_channels = 4; } -message NewAudioSourceResponse { AudioSourceInfo source = 1; } +message NewAudioSourceResponse { OwnedAudioSource source = 1; } // Push a frame to an AudioSource +// The data provided must be available as long as the client receive the callback. message CaptureAudioFrameRequest { uint64 source_handle = 1; - uint64 buffer_handle = 2; + AudioFrameBufferInfo buffer = 2; +} +message CaptureAudioFrameResponse { + uint64 async_id = 1; +} +message CaptureAudioFrameCallback { + uint64 async_id = 1; + optional string error = 2; } -message CaptureAudioFrameResponse {} // Create a new AudioResampler message NewAudioResamplerRequest {} message NewAudioResamplerResponse { - AudioResamplerInfo resampler = 1; + OwnedAudioResampler resampler = 1; } // Remix and resample an audio frame message RemixAndResampleRequest { uint64 resampler_handle = 1; - uint64 buffer_handle = 2; + AudioFrameBufferInfo buffer = 2; uint32 num_channels = 3; uint32 sample_rate = 4; } message RemixAndResampleResponse { - AudioFrameBufferInfo buffer = 1; + OwnedAudioFrameBuffer buffer = 1; } // @@ -74,11 +83,15 @@ message RemixAndResampleResponse { // message AudioFrameBufferInfo { + uint64 data_ptr = 1; // *const i16 + uint32 num_channels = 2; + uint32 sample_rate = 3; + uint32 samples_per_channel = 4; +} + +message OwnedAudioFrameBuffer { FfiOwnedHandle handle = 1; - uint64 data_ptr = 2; // *const i16 - uint32 num_channels = 3; - uint32 sample_rate = 4; - uint32 samples_per_channel = 5; + AudioFrameBufferInfo info = 2; } // @@ -91,8 +104,12 @@ enum AudioStreamType { } message AudioStreamInfo { + AudioStreamType type = 1; +} + +message OwnedAudioStream { FfiOwnedHandle handle = 1; - AudioStreamType type = 2; + AudioStreamInfo info = 2; } message AudioStreamEvent { @@ -101,7 +118,7 @@ message AudioStreamEvent { } message AudioFrameReceived { - AudioFrameBufferInfo frame = 1; + OwnedAudioFrameBuffer frame = 1; } // @@ -119,14 +136,21 @@ enum AudioSourceType { } message AudioSourceInfo { - FfiOwnedHandle handle = 1; AudioSourceType type = 2; } +message OwnedAudioSource { + FfiOwnedHandle handle = 1; + AudioSourceInfo info = 2; +} + // // AudioResampler // -message AudioResamplerInfo { +message AudioResamplerInfo { } + +message OwnedAudioResampler { FfiOwnedHandle handle = 1; + AudioResamplerInfo info = 2; } \ No newline at end of file diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index fdbd672d..500e99da 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -137,6 +137,7 @@ message FfiEvent { PublishTrackCallback publish_track = 8; UnpublishTrackCallback unpublish_track = 9; PublishDataCallback publish_data = 10; + CaptureAudioFrameCallback capture_audio_frame = 11; } } diff --git a/livekit-ffi/protocol/participant.proto b/livekit-ffi/protocol/participant.proto index fcd24d2b..271eccfa 100644 --- a/livekit-ffi/protocol/participant.proto +++ b/livekit-ffi/protocol/participant.proto @@ -20,9 +20,13 @@ option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; message ParticipantInfo { + string sid = 1; + string name = 2; + string identity = 3; + string metadata = 4; +} + +message OwnedParticipant { FfiOwnedHandle handle = 1; - string sid = 2; - string name = 3; - string identity = 4; - string metadata = 5; + ParticipantInfo info = 2; } \ No newline at end of file diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto index dbaf8456..5b8973f2 100644 --- a/livekit-ffi/protocol/room.proto +++ b/livekit-ffi/protocol/room.proto @@ -33,17 +33,17 @@ message ConnectResponse { } message ConnectCallback { message ParticipantWithTracks { - ParticipantInfo participant = 1; + OwnedParticipant participant = 1; // TrackInfo are not needed here, if we're subscribed to a track, the FfiServer will send // a TrackSubscribed event - repeated TrackPublicationInfo publications = 2; + repeated OwnedTrackPublication publications = 2; } uint64 async_id = 1; optional string error = 2; - RoomInfo room = 3; - ParticipantInfo local_participant = 4; + OwnedRoom room = 3; + OwnedParticipant local_participant = 4; repeated ParticipantWithTracks participants = 5; } @@ -64,7 +64,7 @@ message PublishTrackResponse { message PublishTrackCallback { uint64 async_id = 1; optional string error = 2; - TrackPublicationInfo publication = 3; + OwnedTrackPublication publication = 3; } // Unpublish a track from the room @@ -157,9 +157,13 @@ enum DataPacketKind { } message BufferInfo { + uint64 data_ptr = 1; + uint64 data_len = 2; +} + +message OwnedBuffer { FfiOwnedHandle handle = 1; - uint64 data_ptr = 2; - uint64 data_len = 3; + BufferInfo data = 2; } message RoomEvent { @@ -180,7 +184,7 @@ message RoomEvent { ConnectionQualityChanged connection_quality_changed = 14; DataReceived data_received = 15; ConnectionStateChanged connection_state_changed = 16; - Connected connected = 17; + // Connected connected = 17; Disconnected disconnected = 18; Reconnecting reconnecting = 19; Reconnected reconnected = 20; @@ -188,13 +192,17 @@ message RoomEvent { } message RoomInfo { + string sid = 1; + string name = 2; + string metadata = 3; +} + +message OwnedRoom { FfiOwnedHandle handle = 1; - string sid = 2; - string name = 3; - string metadata = 4; + RoomInfo info = 2; } -message ParticipantConnected { ParticipantInfo info = 1; } +message ParticipantConnected { OwnedParticipant info = 1; } message ParticipantDisconnected { string participant_sid = 1; @@ -212,7 +220,7 @@ message LocalTrackUnpublished { message TrackPublished { string participant_sid = 1; - TrackPublicationInfo publication = 2; + OwnedTrackPublication publication = 2; } message TrackUnpublished { @@ -224,7 +232,7 @@ message TrackUnpublished { // The FFI will retrieve the publication using the Track sid message TrackSubscribed { string participant_sid = 1; - TrackInfo track = 2; + OwnedTrack track = 2; } message TrackUnsubscribed { @@ -257,7 +265,7 @@ message ConnectionQualityChanged { } message DataReceived { - BufferInfo data = 1; + OwnedBuffer data = 1; optional string participant_sid = 2; // Can be empty if the data is sent a server SDK DataPacketKind kind = 3; } diff --git a/livekit-ffi/protocol/track.proto b/livekit-ffi/protocol/track.proto index 30b2d7ca..c9a90558 100644 --- a/livekit-ffi/protocol/track.proto +++ b/livekit-ffi/protocol/track.proto @@ -25,7 +25,7 @@ message CreateVideoTrackRequest { uint64 source_handle = 2; } message CreateVideoTrackResponse { - TrackInfo track = 1; + OwnedTrack track = 1; } // Create a new AudioTrack from a AudioSource @@ -34,7 +34,7 @@ message CreateAudioTrackRequest { uint64 source_handle = 2; } message CreateAudioTrackResponse { - TrackInfo track = 1; + OwnedTrack track = 1; } // @@ -64,26 +64,33 @@ enum StreamState { } message TrackPublicationInfo { + string sid = 1; + string name = 2; + TrackKind kind = 3; + TrackSource source = 4; + bool simulcasted = 5; + uint32 width = 6; + uint32 height = 7; + string mime_type = 8; + bool muted = 9; + bool remote = 10; +} + +message OwnedTrackPublication { FfiOwnedHandle handle = 1; - string sid = 2; - string name = 3; - TrackKind kind = 4; - TrackSource source = 5; - bool simulcasted = 6; - uint32 width = 7; - uint32 height = 8; - string mime_type = 9; - bool muted = 10; - bool remote = 11; + TrackPublicationInfo info = 2; } message TrackInfo { - FfiOwnedHandle handle = 1; - string sid = 2; - string name = 3; - TrackKind kind = 4; - StreamState stream_state = 5; - bool muted = 6; - bool remote = 7; + string sid = 1; + string name = 2; + TrackKind kind = 3; + StreamState stream_state = 4; + bool muted = 5; + bool remote = 6; } +message OwnedTrack { + FfiOwnedHandle handle = 1; + TrackInfo info = 2; +} \ No newline at end of file diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index 0bdbbcbe..dce0384b 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -25,7 +25,7 @@ message AllocVideoBufferRequest { uint32 width = 2; uint32 height = 3; } -message AllocVideoBufferResponse { VideoFrameBufferInfo buffer = 1; } +message AllocVideoBufferResponse { OwnedVideoFrameBuffer buffer = 1; } // Create a new VideoStream // VideoStream is used to receive video frames from a track @@ -33,7 +33,7 @@ message NewVideoStreamRequest { uint64 track_handle = 1; VideoStreamType type = 2; } -message NewVideoStreamResponse { VideoStreamInfo stream = 1; } +message NewVideoStreamResponse { OwnedVideoStream stream = 1; } // Create a new VideoSource // VideoSource is used to send video frame to a track @@ -41,15 +41,15 @@ message NewVideoSourceRequest { VideoSourceType type = 1; // Used to determine which encodings to use + simulcast layers // Most of the time it corresponds to the source resolution - optional VideoSourceResolution resolution = 2; + VideoSourceResolution resolution = 2; } -message NewVideoSourceResponse { VideoSourceInfo source = 1; } +message NewVideoSourceResponse { OwnedVideoSource source = 1; } // Push a frame to a VideoSource message CaptureVideoFrameRequest { uint64 source_handle = 1; VideoFrameInfo frame = 2; - uint64 buffer_handle = 3; + uint64 buffer_handle = 3; } message CaptureVideoFrameResponse {} @@ -58,11 +58,13 @@ message CaptureVideoFrameResponse {} message ToI420Request { bool flip_y = 1; oneof from { - ARGBBufferInfo argb = 2; - uint64 buffer_handle = 3; + ArgbBufferInfo argb = 2; + uint64 yuv_handle = 3; // Another yuv buffer } } -message ToI420Response { VideoFrameBufferInfo buffer = 1; } +message ToI420Response { + OwnedVideoFrameBuffer buffer = 1; +} // Convert a YUV frame to a RGBA frame // Only I420 is supported atm @@ -115,10 +117,9 @@ enum VideoFrameBufferType { I444 = 4; I010 = 5; NV12 = 6; - WEBGL = 7; } -message ARGBBufferInfo { +message ArgbBufferInfo { uint64 ptr = 1; VideoFormatType format = 2; uint32 stride = 3; @@ -132,17 +133,21 @@ message VideoFrameInfo { } message VideoFrameBufferInfo { - FfiOwnedHandle handle = 1; - VideoFrameBufferType buffer_type = 2; - uint32 width = 3; - uint32 height = 4; + VideoFrameBufferType buffer_type = 1; + uint32 width = 2; + uint32 height = 3; oneof buffer { - PlanarYuvBufferInfo yuv = 5; - BiplanarYuvBufferInfo bi_yuv = 6; - NativeBufferInfo native = 7; + PlanarYuvBufferInfo yuv = 4; + BiplanarYuvBufferInfo bi_yuv = 5; + NativeBufferInfo native = 6; } } +message OwnedVideoFrameBuffer { + FfiOwnedHandle handle = 1; + VideoFrameBufferInfo info = 2; +} + message PlanarYuvBufferInfo { uint32 chroma_width = 1; uint32 chroma_height = 2; @@ -183,8 +188,12 @@ enum VideoStreamType { } message VideoStreamInfo { + VideoStreamType type = 1; +} + +message OwnedVideoStream { FfiOwnedHandle handle = 1; - VideoStreamType type = 2; + VideoStreamInfo info = 2; } message VideoStreamEvent { @@ -194,7 +203,7 @@ message VideoStreamEvent { message VideoFrameReceived { VideoFrameInfo frame = 1; - VideoFrameBufferInfo buffer = 2; + OwnedVideoFrameBuffer buffer = 2; } // @@ -207,10 +216,14 @@ message VideoSourceResolution { } enum VideoSourceType { - VIDEO_SOURCE_NATIVE = 0; + VIDEO_SOURCE_NATIVE = 0; } message VideoSourceInfo { + VideoSourceType type = 1; +} + +message OwnedVideoSource { FfiOwnedHandle handle = 1; - VideoSourceType type = 2; + VideoSourceInfo info = 2; } diff --git a/livekit-ffi/src/conversion/audio_frame.rs b/livekit-ffi/src/conversion/audio_frame.rs index 3c41df2b..e37afde9 100644 --- a/livekit-ffi/src/conversion/audio_frame.rs +++ b/livekit-ffi/src/conversion/audio_frame.rs @@ -28,31 +28,28 @@ impl From for AudioSourceOptions { } } -impl proto::AudioFrameBufferInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, buffer: &AudioFrame) -> Self { +impl From<&AudioFrame<'_>> for proto::AudioFrameBufferInfo { + fn from(frame: &AudioFrame) -> Self { Self { - handle: Some(handle_id), - data_ptr: buffer.data.as_ptr() as u64, - samples_per_channel: buffer.samples_per_channel, - sample_rate: buffer.sample_rate, - num_channels: buffer.num_channels, + data_ptr: frame.data.as_ptr() as u64, + samples_per_channel: frame.samples_per_channel, + sample_rate: frame.sample_rate, + num_channels: frame.num_channels, } } } -impl proto::AudioSourceInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, source: &FfiAudioSource) -> Self { +impl From<&FfiAudioSource> for proto::AudioSourceInfo { + fn from(source: &FfiAudioSource) -> Self { Self { - handle: Some(handle_id), r#type: source.source_type as i32, } } } -impl proto::AudioStreamInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, stream: &FfiAudioStream) -> Self { +impl From<&FfiAudioStream> for proto::AudioStreamInfo { + fn from(stream: &FfiAudioStream) -> Self { Self { - handle: Some(handle_id), r#type: stream.stream_type as i32, } } diff --git a/livekit-ffi/src/conversion/participant.rs b/livekit-ffi/src/conversion/participant.rs index 2f538d84..d9cea130 100644 --- a/livekit-ffi/src/conversion/participant.rs +++ b/livekit-ffi/src/conversion/participant.rs @@ -15,11 +15,10 @@ use crate::proto; use crate::server::room::FfiParticipant; -impl proto::ParticipantInfo { - pub fn from(handle: proto::FfiOwnedHandle, ffi_participant: &FfiParticipant) -> Self { - let participant = &ffi_participant.participant; +impl From<&FfiParticipant> for proto::ParticipantInfo { + fn from(value: &FfiParticipant) -> Self { + let participant = &value.participant; Self { - handle: Some(handle), sid: participant.sid().into(), name: participant.name(), identity: participant.identity().into(), diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs index 717f6eda..cec0d922 100644 --- a/livekit-ffi/src/conversion/room.rs +++ b/livekit-ffi/src/conversion/room.rs @@ -96,11 +96,10 @@ impl From for AudioEncoding { } } -impl proto::RoomInfo { - pub fn from(handle: proto::FfiOwnedHandle, ffi_room: &FfiRoom) -> Self { - let room = &ffi_room.inner.room; +impl From<&FfiRoom> for proto::RoomInfo { + fn from(value: &FfiRoom) -> Self { + let room = &value.inner.room; Self { - handle: Some(handle), sid: room.sid().into(), name: room.name(), metadata: room.metadata(), diff --git a/livekit-ffi/src/conversion/track.rs b/livekit-ffi/src/conversion/track.rs index 9fa419bc..b558c7ef 100644 --- a/livekit-ffi/src/conversion/track.rs +++ b/livekit-ffi/src/conversion/track.rs @@ -18,23 +18,10 @@ use crate::{ }; use livekit::prelude::*; -impl From for proto::TrackSource { - fn from(source: TrackSource) -> proto::TrackSource { - match source { - TrackSource::Unknown => proto::TrackSource::SourceUnknown, - TrackSource::Camera => proto::TrackSource::SourceCamera, - TrackSource::Microphone => proto::TrackSource::SourceMicrophone, - TrackSource::Screenshare => proto::TrackSource::SourceScreenshare, - TrackSource::ScreenshareAudio => proto::TrackSource::SourceScreenshareAudio, - } - } -} - -impl proto::TrackPublicationInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, ffi_publication: &FfiPublication) -> Self { - let publication = &ffi_publication.publication; +impl From<&FfiPublication> for proto::TrackPublicationInfo { + fn from(value: &FfiPublication) -> Self { + let publication = &value.publication; Self { - handle: Some(handle_id), name: publication.name(), sid: publication.sid().to_string(), kind: proto::TrackKind::from(publication.kind()).into(), @@ -49,11 +36,10 @@ impl proto::TrackPublicationInfo { } } -impl proto::TrackInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, ffi_track: &FfiTrack) -> Self { - let track = &ffi_track.track; +impl From<&FfiTrack> for proto::TrackInfo { + fn from(value: &FfiTrack) -> Self { + let track = &value.track; Self { - handle: Some(handle_id), name: track.name(), stream_state: proto::StreamState::from(track.stream_state()).into(), sid: track.sid().to_string(), @@ -93,3 +79,15 @@ impl From for TrackSource { } } } + +impl From for proto::TrackSource { + fn from(source: TrackSource) -> proto::TrackSource { + match source { + TrackSource::Unknown => proto::TrackSource::SourceUnknown, + TrackSource::Camera => proto::TrackSource::SourceCamera, + TrackSource::Microphone => proto::TrackSource::SourceMicrophone, + TrackSource::Screenshare => proto::TrackSource::SourceScreenshare, + TrackSource::ScreenshareAudio => proto::TrackSource::SourceScreenshareAudio, + } + } +} diff --git a/livekit-ffi/src/conversion/video_frame.rs b/livekit-ffi/src/conversion/video_frame.rs index 96b3f4dd..6d590ae2 100644 --- a/livekit-ffi/src/conversion/video_frame.rs +++ b/livekit-ffi/src/conversion/video_frame.rs @@ -41,6 +41,22 @@ impl proto::VideoFrameInfo { } } +impl From<&FfiVideoSource> for proto::VideoSourceInfo { + fn from(source: &FfiVideoSource) -> Self { + Self { + r#type: source.source_type as i32, + } + } +} + +impl From<&FfiVideoStream> for proto::VideoStreamInfo { + fn from(stream: &FfiVideoStream) -> Self { + Self { + r#type: stream.stream_type as i32, + } + } +} + impl From for VideoFormatType { fn from(format: proto::VideoFormatType) -> Self { match format { @@ -84,7 +100,6 @@ impl From for proto::VideoFrameBufferType { VideoFrameBufferType::I444 => Self::I444, VideoFrameBufferType::I010 => Self::I010, VideoFrameBufferType::NV12 => Self::Nv12, - VideoFrameBufferType::WebGl => Self::Webgl, _ => panic!("unsupported buffer type on FFI server"), } } @@ -136,7 +151,7 @@ macro_rules! impl_yuv_into { } }; ($fncname:ident, $buffer:ty, ALPHA) => { - pub fn $fncname(buffer: $buffer) -> Self { + fn $fncname(buffer: $buffer) -> Self { let (data_y, data_u, data_v, data_a) = buffer.data(); let mut proto = impl_yuv_into!(@fields, buffer, data_y, data_u, data_v); proto.stride_a = buffer.strides().3; @@ -145,7 +160,7 @@ macro_rules! impl_yuv_into { } }; ($fncname:ident, $buffer:ty) => { - pub fn $fncname(buffer: $buffer) -> Self { + fn $fncname(buffer: $buffer) -> Self { let (data_y, data_u, data_v) = buffer.data(); impl_yuv_into!(@fields, buffer, data_y, data_u, data_v) } @@ -154,7 +169,7 @@ macro_rules! impl_yuv_into { macro_rules! impl_biyuv_into { ($fncname:ident, $buffer:ty) => { - pub fn $fncname(buffer: $buffer) -> Self { + fn $fncname(buffer: $buffer) -> Self { let (stride_y, stride_uv) = buffer.strides(); let (data_y, data_uv) = buffer.data(); Self { @@ -182,28 +197,9 @@ impl proto::BiplanarYuvBufferInfo { } impl proto::VideoFrameBufferInfo { - pub fn from(handle: proto::FfiOwnedHandle, buffer: impl AsRef) -> Self { - let buffer = buffer.as_ref(); - match buffer.buffer_type() { - #[cfg(not(target_arch = "wasm32"))] - VideoFrameBufferType::Native => Self::from_native(handle, buffer.as_native().unwrap()), - VideoFrameBufferType::I420 => Self::from_i420(handle, buffer.as_i420().unwrap()), - VideoFrameBufferType::I420A => Self::from_i420a(handle, buffer.as_i420a().unwrap()), - VideoFrameBufferType::I422 => Self::from_i422(handle, buffer.as_i422().unwrap()), - VideoFrameBufferType::I444 => Self::from_i444(handle, buffer.as_i444().unwrap()), - VideoFrameBufferType::I010 => Self::from_i010(handle, buffer.as_i010().unwrap()), - VideoFrameBufferType::NV12 => Self::from_nv12(handle, buffer.as_nv12().unwrap()), - _ => panic!("unsupported buffer type on this platform"), - } - } - #[cfg(not(target_arch = "wasm32"))] - pub fn from_native( - handle: proto::FfiOwnedHandle, - buffer: &video_frame::native::NativeBuffer, - ) -> Self { + fn from_native(buffer: &video_frame::native::NativeBuffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::Native.into(), width: buffer.width(), height: buffer.height(), @@ -213,9 +209,8 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i420(handle: proto::FfiOwnedHandle, buffer: &I420Buffer) -> Self { + fn from_i420(buffer: &I420Buffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::I420.into(), width: buffer.width(), height: buffer.height(), @@ -225,9 +220,8 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i420a(handle: proto::FfiOwnedHandle, buffer: &I420ABuffer) -> Self { + fn from_i420a(buffer: &I420ABuffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::I420a.into(), width: buffer.width(), height: buffer.height(), @@ -237,9 +231,8 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i422(handle: proto::FfiOwnedHandle, buffer: &I422Buffer) -> Self { + fn from_i422(buffer: &I422Buffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::I422.into(), width: buffer.width(), height: buffer.height(), @@ -249,9 +242,8 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i444(handle: proto::FfiOwnedHandle, buffer: &I444Buffer) -> Self { + fn from_i444(buffer: &I444Buffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::I444.into(), width: buffer.width(), height: buffer.height(), @@ -261,9 +253,8 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_i010(handle: proto::FfiOwnedHandle, buffer: &I010Buffer) -> Self { + fn from_i010(buffer: &I010Buffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::I010.into(), width: buffer.width(), height: buffer.height(), @@ -273,9 +264,8 @@ impl proto::VideoFrameBufferInfo { } } - pub fn from_nv12(handle: proto::FfiOwnedHandle, buffer: &NV12Buffer) -> Self { + fn from_nv12(buffer: &NV12Buffer) -> Self { Self { - handle: Some(handle), buffer_type: proto::VideoFrameBufferType::Nv12.into(), width: buffer.width(), height: buffer.height(), @@ -286,20 +276,19 @@ impl proto::VideoFrameBufferInfo { } } -impl proto::VideoSourceInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, source: &FfiVideoSource) -> Self { - Self { - handle: Some(handle_id), - r#type: source.source_type as i32, - } - } -} - -impl proto::VideoStreamInfo { - pub fn from(handle_id: proto::FfiOwnedHandle, stream: &FfiVideoStream) -> Self { - Self { - handle: Some(handle_id), - r#type: stream.stream_type as i32, +impl> From for proto::VideoFrameBufferInfo { + fn from(buffer: B) -> Self { + let buffer = buffer.as_ref(); + match buffer.buffer_type() { + #[cfg(not(target_arch = "wasm32"))] + VideoFrameBufferType::Native => Self::from_native(buffer.as_native().unwrap()), + VideoFrameBufferType::I420 => Self::from_i420(buffer.as_i420().unwrap()), + VideoFrameBufferType::I420A => Self::from_i420a(buffer.as_i420a().unwrap()), + VideoFrameBufferType::I422 => Self::from_i422(buffer.as_i422().unwrap()), + VideoFrameBufferType::I444 => Self::from_i444(buffer.as_i444().unwrap()), + VideoFrameBufferType::I010 => Self::from_i010(buffer.as_i010().unwrap()), + VideoFrameBufferType::NV12 => Self::from_nv12(buffer.as_nv12().unwrap()), + _ => panic!("unsupported buffer type on this platform"), } } } diff --git a/livekit-ffi/src/livekit.proto.rs b/livekit-ffi/src/livekit.proto.rs index f2a59f57..974decce 100644 --- a/livekit-ffi/src/livekit.proto.rs +++ b/livekit-ffi/src/livekit.proto.rs @@ -27,7 +27,7 @@ pub struct CreateVideoTrackRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct CreateVideoTrackResponse { #[prost(message, optional, tag="1")] - pub track: ::core::option::Option, + pub track: ::core::option::Option, } /// Create a new AudioTrack from a AudioSource #[allow(clippy::derive_partial_eq_without_eq)] @@ -42,7 +42,7 @@ pub struct CreateAudioTrackRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct CreateAudioTrackResponse { #[prost(message, optional, tag="1")] - pub track: ::core::option::Option, + pub track: ::core::option::Option, } // // Track @@ -55,47 +55,59 @@ pub struct TrackEvent { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct TrackPublicationInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, - #[prost(string, tag="2")] + #[prost(string, tag="1")] pub sid: ::prost::alloc::string::String, - #[prost(string, tag="3")] + #[prost(string, tag="2")] pub name: ::prost::alloc::string::String, - #[prost(enumeration="TrackKind", tag="4")] + #[prost(enumeration="TrackKind", tag="3")] pub kind: i32, - #[prost(enumeration="TrackSource", tag="5")] + #[prost(enumeration="TrackSource", tag="4")] pub source: i32, - #[prost(bool, tag="6")] + #[prost(bool, tag="5")] pub simulcasted: bool, - #[prost(uint32, tag="7")] + #[prost(uint32, tag="6")] pub width: u32, - #[prost(uint32, tag="8")] + #[prost(uint32, tag="7")] pub height: u32, - #[prost(string, tag="9")] + #[prost(string, tag="8")] pub mime_type: ::prost::alloc::string::String, - #[prost(bool, tag="10")] + #[prost(bool, tag="9")] pub muted: bool, - #[prost(bool, tag="11")] + #[prost(bool, tag="10")] pub remote: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct TrackInfo { +pub struct OwnedTrackPublication { #[prost(message, optional, tag="1")] pub handle: ::core::option::Option, - #[prost(string, tag="2")] + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TrackInfo { + #[prost(string, tag="1")] pub sid: ::prost::alloc::string::String, - #[prost(string, tag="3")] + #[prost(string, tag="2")] pub name: ::prost::alloc::string::String, - #[prost(enumeration="TrackKind", tag="4")] + #[prost(enumeration="TrackKind", tag="3")] pub kind: i32, - #[prost(enumeration="StreamState", tag="5")] + #[prost(enumeration="StreamState", tag="4")] pub stream_state: i32, - #[prost(bool, tag="6")] + #[prost(bool, tag="5")] pub muted: bool, - #[prost(bool, tag="7")] + #[prost(bool, tag="6")] pub remote: bool, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedTrack { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum TrackKind { @@ -192,17 +204,23 @@ impl StreamState { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ParticipantInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, - #[prost(string, tag="2")] + #[prost(string, tag="1")] pub sid: ::prost::alloc::string::String, - #[prost(string, tag="3")] + #[prost(string, tag="2")] pub name: ::prost::alloc::string::String, - #[prost(string, tag="4")] + #[prost(string, tag="3")] pub identity: ::prost::alloc::string::String, - #[prost(string, tag="5")] + #[prost(string, tag="4")] pub metadata: ::prost::alloc::string::String, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedParticipant { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} /// Allocate a new VideoFrameBuffer #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -219,7 +237,7 @@ pub struct AllocVideoBufferRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct AllocVideoBufferResponse { #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, + pub buffer: ::core::option::Option, } /// Create a new VideoStream /// VideoStream is used to receive video frames from a track @@ -235,7 +253,7 @@ pub struct NewVideoStreamRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct NewVideoStreamResponse { #[prost(message, optional, tag="1")] - pub stream: ::core::option::Option, + pub stream: ::core::option::Option, } /// Create a new VideoSource /// VideoSource is used to send video frame to a track @@ -253,7 +271,7 @@ pub struct NewVideoSourceRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct NewVideoSourceResponse { #[prost(message, optional, tag="1")] - pub source: ::core::option::Option, + pub source: ::core::option::Option, } /// Push a frame to a VideoSource #[allow(clippy::derive_partial_eq_without_eq)] @@ -287,15 +305,16 @@ pub mod to_i420_request { pub enum From { #[prost(message, tag="2")] Argb(super::ArgbBufferInfo), + /// Another yuv buffer #[prost(uint64, tag="3")] - BufferHandle(u64), + YuvHandle(u64), } } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct ToI420Response { #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, + pub buffer: ::core::option::Option, } /// Convert a YUV frame to a RGBA frame /// Only I420 is supported atm @@ -361,15 +380,13 @@ pub struct VideoFrameInfo { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct VideoFrameBufferInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, - #[prost(enumeration="VideoFrameBufferType", tag="2")] + #[prost(enumeration="VideoFrameBufferType", tag="1")] pub buffer_type: i32, - #[prost(uint32, tag="3")] + #[prost(uint32, tag="2")] pub width: u32, - #[prost(uint32, tag="4")] + #[prost(uint32, tag="3")] pub height: u32, - #[prost(oneof="video_frame_buffer_info::Buffer", tags="5, 6, 7")] + #[prost(oneof="video_frame_buffer_info::Buffer", tags="4, 5, 6")] pub buffer: ::core::option::Option, } /// Nested message and enum types in `VideoFrameBufferInfo`. @@ -377,16 +394,24 @@ pub mod video_frame_buffer_info { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Buffer { - #[prost(message, tag="5")] + #[prost(message, tag="4")] Yuv(super::PlanarYuvBufferInfo), - #[prost(message, tag="6")] + #[prost(message, tag="5")] BiYuv(super::BiplanarYuvBufferInfo), - #[prost(message, tag="7")] + #[prost(message, tag="6")] Native(super::NativeBufferInfo), } } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedVideoFrameBuffer { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct PlanarYuvBufferInfo { #[prost(uint32, tag="1")] pub chroma_width: u32, @@ -435,10 +460,16 @@ pub struct NativeBufferInfo { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct VideoStreamInfo { + #[prost(enumeration="VideoStreamType", tag="1")] + pub r#type: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedVideoStream { #[prost(message, optional, tag="1")] pub handle: ::core::option::Option, - #[prost(enumeration="VideoStreamType", tag="2")] - pub r#type: i32, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -463,7 +494,7 @@ pub struct VideoFrameReceived { #[prost(message, optional, tag="1")] pub frame: ::core::option::Option, #[prost(message, optional, tag="2")] - pub buffer: ::core::option::Option, + pub buffer: ::core::option::Option, } // // VideoSource @@ -480,10 +511,16 @@ pub struct VideoSourceResolution { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct VideoSourceInfo { + #[prost(enumeration="VideoSourceType", tag="1")] + pub r#type: i32, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedVideoSource { #[prost(message, optional, tag="1")] pub handle: ::core::option::Option, - #[prost(enumeration="VideoSourceType", tag="2")] - pub r#type: i32, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] @@ -588,7 +625,6 @@ pub enum VideoFrameBufferType { I444 = 4, I010 = 5, Nv12 = 6, - Webgl = 7, } impl VideoFrameBufferType { /// String value of the enum field names used in the ProtoBuf definition. @@ -604,7 +640,6 @@ impl VideoFrameBufferType { VideoFrameBufferType::I444 => "I444", VideoFrameBufferType::I010 => "I010", VideoFrameBufferType::Nv12 => "NV12", - VideoFrameBufferType::Webgl => "WEBGL", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -617,7 +652,6 @@ impl VideoFrameBufferType { "I444" => Some(Self::I444), "I010" => Some(Self::I010), "NV12" => Some(Self::Nv12), - "WEBGL" => Some(Self::Webgl), _ => None, } } @@ -703,9 +737,9 @@ pub struct ConnectCallback { #[prost(string, optional, tag="2")] pub error: ::core::option::Option<::prost::alloc::string::String>, #[prost(message, optional, tag="3")] - pub room: ::core::option::Option, + pub room: ::core::option::Option, #[prost(message, optional, tag="4")] - pub local_participant: ::core::option::Option, + pub local_participant: ::core::option::Option, #[prost(message, repeated, tag="5")] pub participants: ::prost::alloc::vec::Vec, } @@ -715,11 +749,11 @@ pub mod connect_callback { #[derive(Clone, PartialEq, ::prost::Message)] pub struct ParticipantWithTracks { #[prost(message, optional, tag="1")] - pub participant: ::core::option::Option, + pub participant: ::core::option::Option, /// TrackInfo are not needed here, if we're subscribed to a track, the FfiServer will send /// a TrackSubscribed event #[prost(message, repeated, tag="2")] - pub publications: ::prost::alloc::vec::Vec, + pub publications: ::prost::alloc::vec::Vec, } } /// Disconnect from the a room @@ -766,7 +800,7 @@ pub struct PublishTrackCallback { #[prost(string, optional, tag="2")] pub error: ::core::option::Option<::prost::alloc::string::String>, #[prost(message, optional, tag="3")] - pub publication: ::core::option::Option, + pub publication: ::core::option::Option, } /// Unpublish a track from the room #[allow(clippy::derive_partial_eq_without_eq)] @@ -886,19 +920,25 @@ pub struct RoomOptions { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct BufferInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, - #[prost(uint64, tag="2")] + #[prost(uint64, tag="1")] pub data_ptr: u64, - #[prost(uint64, tag="3")] + #[prost(uint64, tag="2")] pub data_len: u64, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedBuffer { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub data: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct RoomEvent { #[prost(uint64, tag="1")] pub room_handle: u64, - #[prost(oneof="room_event::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20")] + #[prost(oneof="room_event::Message", tags="2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 18, 19, 20")] pub message: ::core::option::Option, } /// Nested message and enum types in `RoomEvent`. @@ -936,8 +976,7 @@ pub mod room_event { DataReceived(super::DataReceived), #[prost(message, tag="16")] ConnectionStateChanged(super::ConnectionStateChanged), - #[prost(message, tag="17")] - Connected(super::Connected), + /// Connected connected = 17; #[prost(message, tag="18")] Disconnected(super::Disconnected), #[prost(message, tag="19")] @@ -949,20 +988,26 @@ pub mod room_event { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct RoomInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, - #[prost(string, tag="2")] + #[prost(string, tag="1")] pub sid: ::prost::alloc::string::String, - #[prost(string, tag="3")] + #[prost(string, tag="2")] pub name: ::prost::alloc::string::String, - #[prost(string, tag="4")] + #[prost(string, tag="3")] pub metadata: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedRoom { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ParticipantConnected { #[prost(message, optional, tag="1")] - pub info: ::core::option::Option, + pub info: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -990,7 +1035,7 @@ pub struct TrackPublished { #[prost(string, tag="1")] pub participant_sid: ::prost::alloc::string::String, #[prost(message, optional, tag="2")] - pub publication: ::core::option::Option, + pub publication: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1008,7 +1053,7 @@ pub struct TrackSubscribed { #[prost(string, tag="1")] pub participant_sid: ::prost::alloc::string::String, #[prost(message, optional, tag="2")] - pub track: ::core::option::Option, + pub track: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1063,7 +1108,7 @@ pub struct ConnectionQualityChanged { #[derive(Clone, PartialEq, ::prost::Message)] pub struct DataReceived { #[prost(message, optional, tag="1")] - pub data: ::core::option::Option, + pub data: ::core::option::Option, /// Can be empty if the data is sent a server SDK #[prost(string, optional, tag="2")] pub participant_sid: ::core::option::Option<::prost::alloc::string::String>, @@ -1197,7 +1242,7 @@ pub struct AllocAudioBufferRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct AllocAudioBufferResponse { #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, + pub buffer: ::core::option::Option, } /// Create a new AudioStream /// AudioStream is used to receive audio frames from a track @@ -1213,7 +1258,7 @@ pub struct NewAudioStreamRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct NewAudioStreamResponse { #[prost(message, optional, tag="1")] - pub stream: ::core::option::Option, + pub stream: ::core::option::Option, } /// Create a new AudioSource #[allow(clippy::derive_partial_eq_without_eq)] @@ -1223,25 +1268,40 @@ pub struct NewAudioSourceRequest { pub r#type: i32, #[prost(message, optional, tag="2")] pub options: ::core::option::Option, + #[prost(uint32, tag="3")] + pub sample_rate: u32, + #[prost(uint32, tag="4")] + pub num_channels: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct NewAudioSourceResponse { #[prost(message, optional, tag="1")] - pub source: ::core::option::Option, + pub source: ::core::option::Option, } /// Push a frame to an AudioSource +/// The data provided must be available as long as the client receive the callback. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CaptureAudioFrameRequest { #[prost(uint64, tag="1")] pub source_handle: u64, - #[prost(uint64, tag="2")] - pub buffer_handle: u64, + #[prost(message, optional, tag="2")] + pub buffer: ::core::option::Option, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct CaptureAudioFrameResponse { + #[prost(uint64, tag="1")] + pub async_id: u64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CaptureAudioFrameCallback { + #[prost(uint64, tag="1")] + pub async_id: u64, + #[prost(string, optional, tag="2")] + pub error: ::core::option::Option<::prost::alloc::string::String>, } /// Create a new AudioResampler #[allow(clippy::derive_partial_eq_without_eq)] @@ -1252,7 +1312,7 @@ pub struct NewAudioResamplerRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct NewAudioResamplerResponse { #[prost(message, optional, tag="1")] - pub resampler: ::core::option::Option, + pub resampler: ::core::option::Option, } /// Remix and resample an audio frame #[allow(clippy::derive_partial_eq_without_eq)] @@ -1260,8 +1320,8 @@ pub struct NewAudioResamplerResponse { pub struct RemixAndResampleRequest { #[prost(uint64, tag="1")] pub resampler_handle: u64, - #[prost(uint64, tag="2")] - pub buffer_handle: u64, + #[prost(message, optional, tag="2")] + pub buffer: ::core::option::Option, #[prost(uint32, tag="3")] pub num_channels: u32, #[prost(uint32, tag="4")] @@ -1271,7 +1331,7 @@ pub struct RemixAndResampleRequest { #[derive(Clone, PartialEq, ::prost::Message)] pub struct RemixAndResampleResponse { #[prost(message, optional, tag="1")] - pub buffer: ::core::option::Option, + pub buffer: ::core::option::Option, } // // AudioFrame buffer @@ -1280,28 +1340,40 @@ pub struct RemixAndResampleResponse { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AudioFrameBufferInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, /// *const i16 - #[prost(uint64, tag="2")] + #[prost(uint64, tag="1")] pub data_ptr: u64, - #[prost(uint32, tag="3")] + #[prost(uint32, tag="2")] pub num_channels: u32, - #[prost(uint32, tag="4")] + #[prost(uint32, tag="3")] pub sample_rate: u32, - #[prost(uint32, tag="5")] + #[prost(uint32, tag="4")] pub samples_per_channel: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] -pub struct AudioStreamInfo { +pub struct OwnedAudioFrameBuffer { #[prost(message, optional, tag="1")] pub handle: ::core::option::Option, - #[prost(enumeration="AudioStreamType", tag="2")] + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AudioStreamInfo { + #[prost(enumeration="AudioStreamType", tag="1")] pub r#type: i32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedAudioStream { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct AudioStreamEvent { #[prost(uint64, tag="1")] pub source_handle: u64, @@ -1321,7 +1393,7 @@ pub mod audio_stream_event { #[derive(Clone, PartialEq, ::prost::Message)] pub struct AudioFrameReceived { #[prost(message, optional, tag="1")] - pub frame: ::core::option::Option, + pub frame: ::core::option::Option, } // // AudioSource @@ -1340,11 +1412,17 @@ pub struct AudioSourceOptions { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AudioSourceInfo { - #[prost(message, optional, tag="1")] - pub handle: ::core::option::Option, #[prost(enumeration="AudioSourceType", tag="2")] pub r#type: i32, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedAudioSource { + #[prost(message, optional, tag="1")] + pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, +} // // AudioResampler // @@ -1352,8 +1430,14 @@ pub struct AudioSourceInfo { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct AudioResamplerInfo { +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct OwnedAudioResampler { #[prost(message, optional, tag="1")] pub handle: ::core::option::Option, + #[prost(message, optional, tag="2")] + pub info: ::core::option::Option, } // // AudioStream @@ -1565,7 +1649,7 @@ pub mod ffi_response { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct FfiEvent { - #[prost(oneof="ffi_event::Message", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10")] + #[prost(oneof="ffi_event::Message", tags="1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11")] pub message: ::core::option::Option, } /// Nested message and enum types in `FfiEvent`. @@ -1593,6 +1677,8 @@ pub mod ffi_event { UnpublishTrack(super::UnpublishTrackCallback), #[prost(message, tag="10")] PublishData(super::PublishDataCallback), + #[prost(message, tag="11")] + CaptureAudioFrame(super::CaptureAudioFrameCallback), } } /// Setup the callback where the foreign language can receive events diff --git a/livekit-ffi/src/server/audio_source.rs b/livekit-ffi/src/server/audio_source.rs index 83ccf07a..681d1299 100644 --- a/livekit-ffi/src/server/audio_source.rs +++ b/livekit-ffi/src/server/audio_source.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::{borrow::Cow, slice}; + use super::FfiHandle; use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; use livekit::webrtc::prelude::*; @@ -28,15 +30,18 @@ impl FfiAudioSource { pub fn setup( server: &'static server::FfiServer, new_source: proto::NewAudioSourceRequest, - ) -> FfiResult { + ) -> FfiResult { let source_type = new_source.r#type(); #[allow(unreachable_patterns)] let source_inner = match source_type { #[cfg(not(target_arch = "wasm32"))] proto::AudioSourceType::AudioSourceNative => { use livekit::webrtc::audio_source::native::NativeAudioSource; - let audio_source = - NativeAudioSource::new(new_source.options.map(Into::into).unwrap_or_default()); + let audio_source = NativeAudioSource::new( + new_source.options.map(Into::into).unwrap_or_default(), + new_source.sample_rate, + new_source.num_channels, + ); RtcAudioSource::Native(audio_source) } _ => { @@ -46,36 +51,65 @@ impl FfiAudioSource { } }; + let handle_id = server.next_id(); let source = Self { - handle_id: server.next_id(), + handle_id, source_type, source: source_inner, }; - let info = proto::AudioSourceInfo::from( - proto::FfiOwnedHandle { - id: source.handle_id, - }, - &source, - ); + let info = proto::AudioSourceInfo::from(&source); server.store_handle(source.handle_id, source); - Ok(info) + + Ok(proto::OwnedAudioSource { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(info), + }) } pub fn capture_frame( &self, server: &'static server::FfiServer, capture: proto::CaptureAudioFrameRequest, - ) -> FfiResult<()> { - match self.source { - #[cfg(not(target_arch = "wasm32"))] - RtcAudioSource::Native(ref source) => { - let frame = server.retrieve_handle::(capture.buffer_handle)?; - source.capture_frame(frame.value()); + ) -> FfiResult { + let Some(buffer) = capture.buffer else { + return Err(FfiError::InvalidRequest("buffer is None".into())); + }; + + let source = self.source.clone(); + let async_id = server.next_id(); + + tokio::spawn(async move { + // The data must be available as long as the client receive the callback. + let data = unsafe { + let len = buffer.num_channels * buffer.samples_per_channel; + slice::from_raw_parts(buffer.data_ptr as *const i16, len as usize) + }; + + match source { + #[cfg(not(target_arch = "wasm32"))] + RtcAudioSource::Native(ref source) => { + let audio_frame = AudioFrame { + data: Cow::Borrowed(data), + sample_rate: buffer.sample_rate, + num_channels: buffer.num_channels, + samples_per_channel: buffer.samples_per_channel, + }; + + let res = source.capture_frame(&audio_frame).await; + let _ = server + .send_event(proto::ffi_event::Message::CaptureAudioFrame( + proto::CaptureAudioFrameCallback { + async_id, + error: res.err().map(|e| e.to_string()), + }, + )) + .await; + } + _ => {} } - _ => {} - } + }); - Ok(()) + Ok(proto::CaptureAudioFrameResponse { async_id }) } } diff --git a/livekit-ffi/src/server/audio_stream.rs b/livekit-ffi/src/server/audio_stream.rs index 551ad432..2831b0e2 100644 --- a/livekit-ffi/src/server/audio_stream.rs +++ b/livekit-ffi/src/server/audio_stream.rs @@ -43,7 +43,7 @@ impl FfiAudioStream { pub fn setup( server: &'static server::FfiServer, new_stream: proto::NewAudioStreamRequest, - ) -> FfiResult { + ) -> FfiResult { let ffi_track = server.retrieve_handle::(new_stream.track_handle)?; let rtc_track = ffi_track.track.rtc_track(); @@ -53,11 +53,12 @@ impl FfiAudioStream { let (close_tx, close_rx) = oneshot::channel(); let stream_type = new_stream.r#type(); + let handle_id = server.next_id(); let audio_stream = match stream_type { #[cfg(not(target_arch = "wasm32"))] proto::AudioStreamType::AudioStreamNative => { let audio_stream = Self { - handle_id: server.next_id(), + handle_id, stream_type, close_tx, }; @@ -65,7 +66,7 @@ impl FfiAudioStream { let native_stream = NativeAudioStream::new(rtc_track); server.async_runtime.spawn(Self::native_audio_stream_task( server, - audio_stream.handle_id, + handle_id, native_stream, close_rx, )); @@ -78,15 +79,14 @@ impl FfiAudioStream { } }?; - // Store the new audio stream and return the info - let info = proto::AudioStreamInfo::from( - proto::FfiOwnedHandle { - id: audio_stream.handle_id, - }, - &audio_stream, - ); - server.store_handle(audio_stream.handle_id, audio_stream); - Ok(info) + // Store AudioStreamInfothe new audio stream and return the info + let info = proto::AudioStreamInfo::from(&audio_stream); + server.store_handle(handle_id, audio_stream); + + Ok(proto::OwnedAudioStream { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(info), + }) } async fn native_audio_stream_task( @@ -106,7 +106,7 @@ impl FfiAudioStream { }; let handle_id = server.next_id(); - let buffer_info = proto::AudioFrameBufferInfo::from(proto::FfiOwnedHandle{ id: handle_id }, &frame); + let buffer_info = proto::AudioFrameBufferInfo::from(&frame); server.store_handle(handle_id, frame); if let Err(err) = server.send_event(proto::ffi_event::Message::AudioStreamEvent( @@ -114,7 +114,10 @@ impl FfiAudioStream { source_handle: stream_handle_id, message: Some(proto::audio_stream_event::Message::FrameReceived( proto::AudioFrameReceived { - frame: Some(buffer_info), + frame: Some(proto::OwnedAudioFrameBuffer { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(buffer_info), + }), }, )), }, diff --git a/livekit-ffi/src/server/mod.rs b/livekit-ffi/src/server/mod.rs index 6b0cdc61..5adbccc1 100644 --- a/livekit-ffi/src/server/mod.rs +++ b/livekit-ffi/src/server/mod.rs @@ -56,7 +56,7 @@ impl FfiHandle for FfiDataBuffer {} // TODO(theomonnom) Wrap these types impl FfiHandle for Arc> {} -impl FfiHandle for AudioFrame {} +impl FfiHandle for AudioFrame<'static> {} impl FfiHandle for BoxVideoFrameBuffer {} pub struct FfiServer { diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 3ddd5bd0..0cd2f64f 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -167,12 +167,14 @@ fn on_create_video_track( track: Track::LocalVideo(video_track), }; - let track_info = proto::TrackInfo::from(proto::FfiOwnedHandle { id: handle_id }, &ffi_track); - + let track_info = proto::TrackInfo::from(&ffi_track); server.store_handle(handle_id, ffi_track); Ok(proto::CreateVideoTrackResponse { - track: Some(track_info), + track: Some(proto::OwnedTrack { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(track_info), + }), }) } @@ -192,12 +194,14 @@ fn on_create_audio_track( handle: handle_id, track: Track::LocalAudio(audio_track), }; - let track_info = proto::TrackInfo::from(proto::FfiOwnedHandle { id: handle_id }, &ffi_track); - + let track_info = proto::TrackInfo::from(&ffi_track); server.store_handle(handle_id, ffi_track); Ok(proto::CreateAudioTrackResponse { - track: Some(track_info), + track: Some(proto::OwnedTrack { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(track_info), + }), }) } @@ -216,12 +220,14 @@ fn on_alloc_video_buffer( }; let handle_id = server.next_id(); - let buffer_info = - proto::VideoFrameBufferInfo::from(proto::FfiOwnedHandle { id: handle_id }, &buffer); - + let buffer_info = proto::VideoFrameBufferInfo::from(&buffer); server.store_handle(handle_id, buffer); + Ok(proto::AllocVideoBufferResponse { - buffer: Some(buffer_info), + buffer: Some(proto::OwnedVideoFrameBuffer { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(buffer_info), + }), }) } @@ -259,6 +265,7 @@ fn on_capture_video_frame( } /// Convert a frame to I420 +/// The destination can now be a new buffer or an existing buffer fn on_to_i420( server: &'static FfiServer, to_i420: proto::ToI420Request, @@ -285,12 +292,10 @@ fn on_to_i420( match info.format() { proto::VideoFormatType::FormatArgb => { - yuv_helper::argb_to_i420(argb, info.stride, dy, sy, du, su, dv, sv, w, h) - .unwrap(); + yuv_helper::argb_to_i420(argb, info.stride, dy, sy, du, su, dv, sv, w, h); } proto::VideoFormatType::FormatAbgr => { - yuv_helper::abgr_to_i420(argb, info.stride, dy, sy, du, su, dv, sv, w, h) - .unwrap(); + yuv_helper::abgr_to_i420(argb, info.stride, dy, sy, du, su, dv, sv, w, h); } _ => { return Err(FfiError::InvalidRequest( @@ -303,18 +308,20 @@ fn on_to_i420( } // Convert an arbitrary yuv buffer to I420 // Even if the buffer is already in I420, we're still copying it - proto::to_i420_request::From::BufferHandle(handle) => server + proto::to_i420_request::From::YuvHandle(handle) => server .retrieve_handle::(handle)? .to_i420(), }; let i420: BoxVideoFrameBuffer = Box::new(i420); let handle_id = server.next_id(); - let buffer_info = - proto::VideoFrameBufferInfo::from(proto::FfiOwnedHandle { id: handle_id }, &i420); + let buffer_info = proto::VideoFrameBufferInfo::from(&i420); server.store_handle(handle_id, i420); Ok(proto::ToI420Response { - buffer: Some(buffer_info), + buffer: Some(proto::OwnedVideoFrameBuffer { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(buffer_info), + }), }) } @@ -336,10 +343,7 @@ fn on_to_argb( let h = to_argb.dst_height as i32 * (if to_argb.flip_y { -1 } else { 1 }); let video_format = to_argb.dst_format(); - buffer - .to_argb(video_format.into(), argb, to_argb.dst_stride, w, h) - .unwrap(); - + buffer.to_argb(video_format.into(), argb, to_argb.dst_stride, w, h); Ok(proto::ToArgbResponse::default()) } @@ -355,12 +359,14 @@ fn on_alloc_audio_buffer( ); let handle_id = server.next_id(); - let frame_info = - proto::AudioFrameBufferInfo::from(proto::FfiOwnedHandle { id: handle_id }, &frame); + let buffer_info = proto::AudioFrameBufferInfo::from(&frame); server.store_handle(handle_id, frame); Ok(proto::AllocAudioBufferResponse { - buffer: Some(frame_info), + buffer: Some(proto::OwnedAudioFrameBuffer { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(buffer_info), + }), }) } @@ -408,8 +414,9 @@ fn new_audio_resampler( server.store_handle(handle_id, resampler); Ok(proto::NewAudioResamplerResponse { - resampler: Some(proto::AudioResamplerInfo { + resampler: Some(proto::OwnedAudioResampler { handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(proto::AudioResamplerInfo {}), }), }) } @@ -423,11 +430,19 @@ fn remix_and_resample( .retrieve_handle::>>(remix.resampler_handle)? .clone(); - let buffer = server.retrieve_handle::(remix.buffer_handle)?; + let buffer = remix + .buffer + .ok_or(FfiError::InvalidRequest("buffer is empty".into()))?; + + let data = unsafe { + let len = (buffer.num_channels * buffer.samples_per_channel) as usize; + slice::from_raw_parts_mut(buffer.data_ptr as *mut i16, len) + }; + let data = resampler .lock() .remix_and_resample( - &buffer.data, + data, buffer.samples_per_channel, buffer.num_channels, buffer.sample_rate, @@ -436,23 +451,23 @@ fn remix_and_resample( ) .to_owned(); - drop(buffer); - let data_len = (data.len() / remix.num_channels as usize) as u32; let audio_frame = AudioFrame { - data, + data: data.into(), num_channels: remix.num_channels, samples_per_channel: data_len, sample_rate: remix.sample_rate, }; let handle_id = server.next_id(); - let buffer_info = - proto::AudioFrameBufferInfo::from(proto::FfiOwnedHandle { id: handle_id }, &audio_frame); + let buffer_info = proto::AudioFrameBufferInfo::from(&audio_frame); server.store_handle(handle_id, audio_frame); Ok(proto::RemixAndResampleResponse { - buffer: Some(buffer_info), + buffer: Some(proto::OwnedAudioFrameBuffer { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(buffer_info), + }), }) } diff --git a/livekit-ffi/src/server/room.rs b/livekit-ffi/src/server/room.rs index b8f2c54c..e6e55c02 100644 --- a/livekit-ffi/src/server/room.rs +++ b/livekit-ffi/src/server/room.rs @@ -105,9 +105,10 @@ impl FfiRoom { let (data_tx, data_rx) = mpsc::unbounded_channel(); let (close_tx, close_rx) = broadcast::channel(1); + let handle_id = server.next_id(); let inner = Arc::new(RoomInner { room, - handle_id: server.next_id(), + handle_id, data_tx, pending_published_tracks: Default::default(), pending_unpublished_tracks: Default::default(), @@ -117,10 +118,6 @@ impl FfiRoom { build_initial_states(server, &inner, participants_with_tracks); // Send callback - let room_handle = proto::FfiOwnedHandle { - id: inner.handle_id, - }; - let ffi_room = Self { inner: inner.clone(), handle: Default::default(), @@ -130,6 +127,7 @@ impl FfiRoom { // Keep the lock until the handle is "Some" (So it is OK for the client to request a disconnect quickly after connecting) // (When requesting a disconnect, the handle will still be locked and the disconnect will wait for the lock to be released and gracefully close the room) let mut handle = ffi_room.handle.lock().await; + let room_info = proto::RoomInfo::from(&ffi_room); // Send the async response to the FfiClient *before* starting the tasks. // Ensure no events are sent before the callback @@ -137,7 +135,10 @@ impl FfiRoom { .send_event(proto::ffi_event::Message::Connect(proto::ConnectCallback { async_id, error: None, - room: Some(proto::RoomInfo::from(room_handle, &ffi_room)), + room: Some(proto::OwnedRoom { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(room_info), + }), local_participant: Some(local_info), participants: remote_infos, })) @@ -245,25 +246,23 @@ impl RoomInner { match publish_res { Ok(publication) => { // Successfully published the track + let handle_id = server.next_id(); let ffi_publication = FfiPublication { - handle: server.next_id(), + handle: handle_id, publication: TrackPublication::Local(publication.clone()), }; - let publication_info = proto::TrackPublicationInfo::from( - proto::FfiOwnedHandle { - id: ffi_publication.handle, - }, - &ffi_publication, - ); - + let publication_info = proto::TrackPublicationInfo::from(&ffi_publication); server.store_handle(ffi_publication.handle, ffi_publication); let _ = server .send_event(proto::ffi_event::Message::PublishTrack( proto::PublishTrackCallback { async_id, - publication: Some(publication_info), + publication: Some(proto::OwnedTrackPublication { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(publication_info), + }), ..Default::default() }, )) @@ -399,10 +398,10 @@ async fn forward_event(server: &'static FfiServer, inner: &Arc, event let _ = send_event(proto::room_event::Message::ParticipantConnected( proto::ParticipantConnected { - info: Some(proto::ParticipantInfo::from( - proto::FfiOwnedHandle { id: handle_id }, - &ffi_participant, - )), + info: Some(proto::OwnedParticipant { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(proto::ParticipantInfo::from(&ffi_participant)), + }), }, )) .await; @@ -463,24 +462,22 @@ async fn forward_event(server: &'static FfiServer, inner: &Arc, event publication, participant, } => { + let handle_id = server.next_id(); let ffi_publication = FfiPublication { - handle: server.next_id(), + handle: handle_id, publication: TrackPublication::Remote(publication), }; - let publication_info = proto::TrackPublicationInfo::from( - proto::FfiOwnedHandle { - id: ffi_publication.handle, - }, - &ffi_publication, - ); - + let publication_info = proto::TrackPublicationInfo::from(&ffi_publication); server.store_handle(ffi_publication.handle, ffi_publication); let _ = send_event(proto::room_event::Message::TrackPublished( proto::TrackPublished { participant_sid: participant.sid().to_string(), - publication: Some(publication_info), + publication: Some(proto::OwnedTrackPublication { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(publication_info), + }), }, )) .await; @@ -502,24 +499,22 @@ async fn forward_event(server: &'static FfiServer, inner: &Arc, event publication: _, participant, } => { + let handle_id = server.next_id(); let ffi_track = FfiTrack { - handle: server.next_id(), + handle: handle_id, track: track.into(), }; - let track_info = proto::TrackInfo::from( - proto::FfiOwnedHandle { - id: ffi_track.handle, - }, - &ffi_track, - ); - + let track_info = proto::TrackInfo::from(&ffi_track); server.store_handle(ffi_track.handle, ffi_track); let _ = send_event(proto::room_event::Message::TrackSubscribed( proto::TrackSubscribed { participant_sid: participant.sid().to_string(), - track: Some(track_info), + track: Some(proto::OwnedTrack { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(track_info), + }), }, )) .await; @@ -601,23 +596,25 @@ async fn forward_event(server: &'static FfiServer, inner: &Arc, event kind, participant, } => { - let next_id = server.next_id(); + let handle_id = server.next_id(); let buffer_info = proto::BufferInfo { - handle: Some(proto::FfiOwnedHandle { id: next_id }), data_ptr: payload.as_ptr() as u64, data_len: payload.len() as u64, }; server.store_handle( - next_id, + handle_id, FfiDataBuffer { - handle: next_id, + handle: handle_id, data: payload, }, ); let _ = send_event(proto::room_event::Message::DataReceived( proto::DataReceived { - data: Some(buffer_info), + data: Some(proto::OwnedBuffer { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + data: Some(buffer_info), + }), participant_sid: Some(participant.sid().to_string()), kind: proto::DataPacketKind::from(kind).into(), }, @@ -662,63 +659,65 @@ fn build_initial_states( inner: &Arc, participants_with_tracks: Vec<(RemoteParticipant, Vec)>, ) -> ( - proto::ParticipantInfo, + proto::OwnedParticipant, Vec, ) { let local_participant = inner.room.local_participant(); // Is it too late to get the local participant info here? + let handle_id = server.next_id(); let local_participant = FfiParticipant { - handle: server.next_id(), + handle: handle_id, participant: Participant::Local(local_participant), room: inner.clone(), }; - server.store_handle(local_participant.handle, local_participant.clone()); - let local_info = proto::ParticipantInfo::from( - proto::FfiOwnedHandle { - id: local_participant.handle, - }, - &local_participant, - ); + let local_info = proto::ParticipantInfo::from(&local_participant); + server.store_handle(local_participant.handle, local_participant); let remote_infos = participants_with_tracks .into_iter() .map(|(participant, tracks)| { + let handle_id = server.next_id(); let ffi_participant = FfiParticipant { - handle: server.next_id(), + handle: handle_id, participant: Participant::Remote(participant), room: inner.clone(), }; - server.store_handle(ffi_participant.handle, ffi_participant.clone()); - let remote_info = proto::ParticipantInfo::from( - proto::FfiOwnedHandle { - id: ffi_participant.handle, - }, - &ffi_participant, - ); + let remote_info = proto::ParticipantInfo::from(&ffi_participant); + server.store_handle(ffi_participant.handle, ffi_participant); proto::connect_callback::ParticipantWithTracks { - participant: Some(remote_info), + participant: Some(proto::OwnedParticipant { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(remote_info), + }), publications: tracks .into_iter() .map(|track| { + let handle_id = server.next_id(); let ffi_publication = FfiPublication { - handle: server.next_id(), + handle: handle_id, publication: TrackPublication::Remote(track), }; - server.store_handle(ffi_publication.handle, ffi_publication.clone()); - proto::TrackPublicationInfo::from( - proto::FfiOwnedHandle { - id: ffi_publication.handle, - }, - &ffi_publication, - ) + let track_info = proto::TrackPublicationInfo::from(&ffi_publication); + server.store_handle(ffi_publication.handle, ffi_publication); + + proto::OwnedTrackPublication { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(track_info), + } }) .collect::>(), } }) .collect::>(); - (local_info, remote_infos) + ( + proto::OwnedParticipant { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(local_info), + }, + remote_infos, + ) } diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs index 60b87b84..1574fe6d 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -30,7 +30,7 @@ impl FfiVideoSource { pub fn setup( server: &'static server::FfiServer, new_source: proto::NewVideoSourceRequest, - ) -> FfiResult { + ) -> FfiResult { let source_type = new_source.r#type(); #[allow(unreachable_patterns)] let source_inner = match source_type { @@ -49,20 +49,19 @@ impl FfiVideoSource { } }; + let handle_id = server.next_id(); let video_source = Self { - handle_id: server.next_id(), + handle_id, source_type, source: source_inner, }; - let source_info = proto::VideoSourceInfo::from( - proto::FfiOwnedHandle { - id: video_source.handle_id, - }, - &video_source, - ); + let source_info = proto::VideoSourceInfo::from(&video_source); + server.store_handle(handle_id, video_source); - server.store_handle(video_source.handle_id, video_source); - Ok(source_info) + Ok(proto::OwnedVideoSource { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(source_info), + }) } pub fn capture_frame( diff --git a/livekit-ffi/src/server/video_stream.rs b/livekit-ffi/src/server/video_stream.rs index 3b396d3c..de216def 100644 --- a/livekit-ffi/src/server/video_stream.rs +++ b/livekit-ffi/src/server/video_stream.rs @@ -43,7 +43,7 @@ impl FfiVideoStream { pub fn setup( server: &'static server::FfiServer, new_stream: proto::NewVideoStreamRequest, - ) -> FfiResult { + ) -> FfiResult { let ffi_track = server.retrieve_handle::(new_stream.track_handle)?; let rtc_track = ffi_track.track.rtc_track(); @@ -53,17 +53,18 @@ impl FfiVideoStream { let (close_tx, close_rx) = oneshot::channel(); let stream_type = new_stream.r#type(); + let handle_id = server.next_id(); let stream = match stream_type { #[cfg(not(target_arch = "wasm32"))] proto::VideoStreamType::VideoStreamNative => { let video_stream = Self { - handle_id: server.next_id(), + handle_id, close_tx, stream_type, }; server.async_runtime.spawn(Self::native_video_stream_task( server, - video_stream.handle_id, + handle_id, NativeVideoStream::new(rtc_track), close_rx, )); @@ -77,14 +78,13 @@ impl FfiVideoStream { }?; // Store the new video stream and return the info - let info = proto::VideoStreamInfo::from( - proto::FfiOwnedHandle { - id: stream.handle_id, - }, - &stream, - ); + let info = proto::VideoStreamInfo::from(&stream); server.store_handle(stream.handle_id, stream); - Ok(info) + + Ok(proto::OwnedVideoStream { + handle: Some(proto::FfiOwnedHandle { id: handle_id }), + info: Some(info), + }) } async fn native_video_stream_task( @@ -105,7 +105,7 @@ impl FfiVideoStream { let handle_id = server.next_id(); let frame_info = proto::VideoFrameInfo::from(&frame); - let buffer_info = proto::VideoFrameBufferInfo::from(proto::FfiOwnedHandle { id: handle_id }, &frame.buffer); + let buffer_info = proto::VideoFrameBufferInfo::from(&frame.buffer); server.store_handle(handle_id, frame.buffer); if let Err(err) = server.send_event(proto::ffi_event::Message::VideoStreamEvent( @@ -114,7 +114,12 @@ impl FfiVideoStream { message: Some(proto::video_stream_event::Message::FrameReceived( proto::VideoFrameReceived { frame: Some(frame_info), - buffer: Some(buffer_info), + buffer: Some(proto::OwnedVideoFrameBuffer { + handle: Some(proto::FfiOwnedHandle { + id: handle_id, + }), + info: Some(buffer_info), + }), } )), } diff --git a/livekit-webrtc/src/audio_frame.rs b/livekit-webrtc/src/audio_frame.rs index a3c537b1..de0a39c5 100644 --- a/livekit-webrtc/src/audio_frame.rs +++ b/livekit-webrtc/src/audio_frame.rs @@ -12,18 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Cow; + #[derive(Debug, Clone)] -pub struct AudioFrame { - pub data: Vec, +pub struct AudioFrame<'a> { + pub data: Cow<'a, [i16]>, pub sample_rate: u32, pub num_channels: u32, pub samples_per_channel: u32, } -impl AudioFrame { +impl AudioFrame<'_> { + // Owned pub fn new(sample_rate: u32, num_channels: u32, samples_per_channel: u32) -> Self { Self { - data: vec![0; (num_channels * samples_per_channel) as usize], + data: vec![0; (num_channels * samples_per_channel) as usize].into(), sample_rate, num_channels, samples_per_channel, diff --git a/livekit-webrtc/src/audio_source.rs b/livekit-webrtc/src/audio_source.rs index 3ba8e400..ac35a46a 100644 --- a/livekit-webrtc/src/audio_source.rs +++ b/livekit-webrtc/src/audio_source.rs @@ -40,7 +40,7 @@ impl RtcAudioSource { #[cfg(not(target_arch = "wasm32"))] pub mod native { use super::*; - use crate::audio_frame::AudioFrame; + use crate::{audio_frame::AudioFrame, RtcError}; use std::fmt::{Debug, Formatter}; #[derive(Clone)] @@ -54,21 +54,19 @@ pub mod native { } } - impl Default for NativeAudioSource { - fn default() -> Self { - Self::new(AudioSourceOptions::default()) - } - } - impl NativeAudioSource { - pub fn new(options: AudioSourceOptions) -> NativeAudioSource { + pub fn new( + options: AudioSourceOptions, + sample_rate: u32, + num_channels: u32, + ) -> NativeAudioSource { Self { - handle: imp_as::NativeAudioSource::new(options), + handle: imp_as::NativeAudioSource::new(options, sample_rate, num_channels), } } - pub fn capture_frame(&self, frame: &AudioFrame) { - self.handle.capture_frame(frame) + pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> { + self.handle.capture_frame(frame).await } pub fn set_audio_options(&self, options: AudioSourceOptions) { @@ -78,5 +76,13 @@ pub mod native { pub fn audio_options(&self) -> AudioSourceOptions { self.handle.audio_options() } + + pub fn sample_rate(&self) -> u32 { + self.handle.sample_rate() + } + + pub fn num_channels(&self) -> u32 { + self.handle.num_channels() + } } } diff --git a/livekit-webrtc/src/audio_stream.rs b/livekit-webrtc/src/audio_stream.rs index d472c754..109d3ada 100644 --- a/livekit-webrtc/src/audio_stream.rs +++ b/livekit-webrtc/src/audio_stream.rs @@ -53,7 +53,7 @@ pub mod native { } impl Stream for NativeAudioStream { - type Item = AudioFrame; + type Item = AudioFrame<'static>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context) -> Poll> { Pin::new(&mut self.get_mut().handle).poll_next(cx) diff --git a/livekit-webrtc/src/native/audio_resampler.rs b/livekit-webrtc/src/native/audio_resampler.rs index b2e01733..13b43e57 100644 --- a/livekit-webrtc/src/native/audio_resampler.rs +++ b/livekit-webrtc/src/native/audio_resampler.rs @@ -28,15 +28,20 @@ impl Default for AudioResampler { } impl AudioResampler { - pub fn remix_and_resample( - &mut self, + pub fn remix_and_resample<'a>( + &'a mut self, src: &[i16], samples_per_channel: u32, num_channels: u32, sample_rate: u32, dst_num_channels: u32, dst_sample_rate: u32, - ) -> &[i16] { + ) -> &'a [i16] { + assert!( + src.len() >= (samples_per_channel * num_channels) as usize, + "src buffer too small" + ); + unsafe { let len = self.sys_handle.pin_mut().remix_and_resample( src.as_ptr(), diff --git a/livekit-webrtc/src/native/audio_source.rs b/livekit-webrtc/src/native/audio_source.rs index 91c6b035..cf507d77 100644 --- a/livekit-webrtc/src/native/audio_source.rs +++ b/livekit-webrtc/src/native/audio_source.rs @@ -12,51 +12,56 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions}; +use crate::{audio_frame::AudioFrame, audio_source::AudioSourceOptions, RtcError, RtcErrorType}; use cxx::SharedPtr; -use parking_lot::Mutex; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; +use tokio::{ + sync::{Mutex as AsyncMutex, MutexGuard}, + time::interval, +}; use webrtc_sys::audio_track as sys_at; -impl From for AudioSourceOptions { - fn from(options: sys_at::ffi::AudioSourceOptions) -> Self { - Self { - echo_cancellation: options.echo_cancellation, - noise_suppression: options.noise_suppression, - auto_gain_control: options.auto_gain_control, - } - } -} - -impl From for sys_at::ffi::AudioSourceOptions { - fn from(options: AudioSourceOptions) -> Self { - Self { - echo_cancellation: options.echo_cancellation, - noise_suppression: options.noise_suppression, - auto_gain_control: options.auto_gain_control, - } - } -} - #[derive(Clone)] pub struct NativeAudioSource { sys_handle: SharedPtr, - inner: Arc>, + inner: Arc>, + sample_rate: u32, + num_channels: u32, + samples_10ms: usize, } -#[derive(Default)] struct AudioSourceInner { - buf: Vec, - offset: usize, - sample_rate: u32, - num_channels: u32, + buf: Box<[i16]>, + + // Amount of data from the previous frame that hasn't been sent to the libwebrtc source + // (because it requires 10ms of data) + len: usize, + + // Amount of data that have been read inside the current AudioFrame + read_offset: usize, + + interval: Option, } impl NativeAudioSource { - pub fn new(options: AudioSourceOptions) -> NativeAudioSource { + pub fn new( + options: AudioSourceOptions, + sample_rate: u32, + num_channels: u32, + ) -> NativeAudioSource { + let samples_10ms = (sample_rate / 100 * num_channels) as usize; + Self { sys_handle: sys_at::ffi::new_audio_track_source(options.into()), - inner: Default::default(), + inner: Arc::new(AsyncMutex::new(AudioSourceInner { + buf: vec![0; samples_10ms].into_boxed_slice(), + len: 0, + read_offset: 0, + interval: Some(interval(Duration::from_millis(10))), + })), + sample_rate, + num_channels, + samples_10ms, } } @@ -73,52 +78,195 @@ impl NativeAudioSource { self.sys_handle.audio_options().into() } - pub fn capture_frame(&self, frame: &AudioFrame) { - let mut inner = self.inner.lock(); - let samples_10ms = (frame.sample_rate / 100 * frame.num_channels) as usize; - if inner.sample_rate != frame.sample_rate || inner.num_channels != frame.num_channels { - inner.buf.resize(samples_10ms as usize, 0); - inner.offset = 0; - inner.sample_rate = frame.sample_rate; - inner.num_channels = frame.num_channels; - } - - // Split the frame into 10ms chunks - let mut i = 0; - loop { - let buf_offset = inner.offset; - let remaining_data = frame.data.len() - i; // Remaining data to read inside the frame - let needed_data = samples_10ms - buf_offset; // Needed data of "frame.data" to make a complete 10ms from inner.buf - if remaining_data < needed_data { - if remaining_data > 0 { - // Not enough data to make a complete 10ms frame, store the remaining data inside inner.buf - // It'll be used on the next capture. - inner.buf[buf_offset..buf_offset + remaining_data] - .copy_from_slice(&frame.data[i..]); - inner.offset += remaining_data; - } + pub fn sample_rate(&self) -> u32 { + self.sample_rate + } - break; - } + pub fn num_channels(&self) -> u32 { + self.num_channels + } - let data = if inner.offset != 0 { - // Use the data from the previous capture - let data = &mut inner.buf[buf_offset..]; - data.copy_from_slice(&frame.data[i..i + needed_data]); - inner.offset = 0; + // Implemented inside another functions to allow unit testing + fn next_frame<'a>( + &self, + inner: &'a mut MutexGuard<'_, AudioSourceInner>, // The lock musts be guarded by capture_frame + frame: &'a AudioFrame<'a>, + ) -> Option<&'a [i16]> { + let available_data = inner.len + frame.data.len() - inner.read_offset; + if available_data >= self.samples_10ms { + Some(if inner.len != 0 { + // Read 10ms frame from inner.buf AND frame.data + let missing_len = self.samples_10ms - inner.len; + let start = inner.len; + inner.buf[start..].copy_from_slice(&frame.data[..missing_len]); + inner.read_offset += missing_len; + inner.len = 0; &inner.buf } else { - &frame.data[i..i + samples_10ms] + // Read 10ms frame only from frame.data + let start = inner.read_offset; + let end = start + self.samples_10ms; + inner.read_offset += self.samples_10ms; + &frame.data[start..end] + }) + } else { + // Save to buf and wait for the next capture_frame to give enough data to complete a 10ms frame + let remaining_data = frame.data.len() - inner.read_offset; // remaining data from frame.data + let start = inner.len; + let end = start + remaining_data; + let start2 = frame.data.len() - remaining_data; + inner.buf[start..end].copy_from_slice(&frame.data[start2..]); + inner.len += remaining_data; + inner.read_offset = 0; + None + } + } + + pub async fn capture_frame(&self, frame: &AudioFrame<'_>) -> Result<(), RtcError> { + if self.sample_rate != frame.sample_rate || self.num_channels != frame.num_channels { + return Err(RtcError { + error_type: RtcErrorType::InvalidState, + message: "sample_rate and num_channels don't match".to_owned(), + }); + } + + let mut inner = self.inner.lock().await; + let mut interval = inner.interval.take().unwrap(); // How can I avoid double mut borrow? + + loop { + let Some(data) = self.next_frame(&mut inner, frame) else { + inner.interval = Some(interval); + break; }; + interval.tick().await; + + let samples_per_channel = data.len() / self.num_channels as usize; self.sys_handle.on_captured_frame( data, - frame.sample_rate as i32, - frame.num_channels as usize, - samples_10ms / frame.num_channels as usize, + self.sample_rate as i32, + self.num_channels as usize, + samples_per_channel, ); + } + + Ok(()) + } +} + +impl From for AudioSourceOptions { + fn from(options: sys_at::ffi::AudioSourceOptions) -> Self { + Self { + echo_cancellation: options.echo_cancellation, + noise_suppression: options.noise_suppression, + auto_gain_control: options.auto_gain_control, + } + } +} - i += needed_data; +impl From for sys_at::ffi::AudioSourceOptions { + fn from(options: AudioSourceOptions) -> Self { + Self { + echo_cancellation: options.echo_cancellation, + noise_suppression: options.noise_suppression, + auto_gain_control: options.auto_gain_control, } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn split_frames() { + let source = NativeAudioSource::new(AudioSourceOptions::default(), 48000, 2); + let samples_count = + source.sample_rate() as usize / 1000 * 20 * source.num_channels() as usize; // 20ms + + let audio_frame = AudioFrame { + data: vec![0; samples_count].into(), + sample_rate: source.sample_rate(), + num_channels: source.num_channels(), + samples_per_channel: samples_count as u32 / source.num_channels(), + }; + + let mut inner = source.inner.lock().await; + + assert!(source.next_frame(&mut inner, &audio_frame).is_some()); + assert!(source.next_frame(&mut inner, &audio_frame).is_some()); + assert!(source.next_frame(&mut inner, &audio_frame).is_none()); + } + + #[tokio::test] + async fn buffer_is_used() { + let source = NativeAudioSource::new(AudioSourceOptions::default(), 48000, 2); + let samples_15ms = + source.sample_rate() as usize / 1000 * 15 * source.num_channels() as usize; + + let audio_frame = AudioFrame { + data: vec![0; samples_15ms].into(), + sample_rate: source.sample_rate(), + num_channels: source.num_channels(), + samples_per_channel: samples_15ms as u32 / source.num_channels(), + }; + + let mut inner = source.inner.lock().await; + + assert!(source.next_frame(&mut inner, &audio_frame).is_some()); + assert!(source.next_frame(&mut inner, &audio_frame).is_none()); + + let samples_5ms = source.sample_rate() as usize / 1000 * 5 * source.num_channels() as usize; + assert_eq!(inner.len, samples_5ms); // Remains 5ms + + let samples_12ms = + source.sample_rate() as usize / 1000 * 12 * source.num_channels() as usize; + + let audio_frame = AudioFrame { + data: vec![0; samples_12ms].into(), + sample_rate: source.sample_rate(), + num_channels: source.num_channels(), + samples_per_channel: samples_12ms as u32 / source.num_channels(), + }; + + assert!(source.next_frame(&mut inner, &audio_frame).is_some()); + assert!(source.next_frame(&mut inner, &audio_frame).is_none()); + + let samples_7ms = source.sample_rate() as usize / 1000 * 7 * source.num_channels() as usize; + assert_eq!(inner.len, samples_7ms); // Remains 7ms + } + + #[tokio::test] + async fn verify_duration() { + let source = NativeAudioSource::new(AudioSourceOptions::default(), 48000, 2); + let samples_30ms = + source.sample_rate() as usize / 1000 * 35 * source.num_channels() as usize; + + let audio_frame = AudioFrame { + data: vec![0; samples_30ms].into(), + sample_rate: source.sample_rate(), + num_channels: source.num_channels(), + samples_per_channel: samples_30ms as u32 / source.num_channels(), + }; + + let mut inner = source.inner.lock().await; + + let samples_10ms = source.sample_rate() as usize / 100 * source.num_channels() as usize; + assert_eq!( + source.next_frame(&mut inner, &audio_frame).unwrap().len(), + samples_10ms + ); + assert_eq!( + source.next_frame(&mut inner, &audio_frame).unwrap().len(), + samples_10ms + ); + assert_eq!( + source.next_frame(&mut inner, &audio_frame).unwrap().len(), + samples_10ms + ); + assert!(source.next_frame(&mut inner, &audio_frame).is_none()); + + let samples_5ms = source.sample_rate() as usize / 1000 * 5 * source.num_channels() as usize; + assert_eq!(inner.len, samples_5ms); // Remaining data + } +} diff --git a/livekit-webrtc/src/native/audio_stream.rs b/livekit-webrtc/src/native/audio_stream.rs index a3f53090..1c730beb 100644 --- a/livekit-webrtc/src/native/audio_stream.rs +++ b/livekit-webrtc/src/native/audio_stream.rs @@ -25,7 +25,7 @@ use webrtc_sys::audio_track as sys_at; pub struct NativeAudioStream { native_sink: SharedPtr, audio_track: RtcAudioTrack, - frame_rx: mpsc::UnboundedReceiver, + frame_rx: mpsc::UnboundedReceiver>, } impl NativeAudioStream { @@ -65,7 +65,7 @@ impl Drop for NativeAudioStream { } impl Stream for NativeAudioStream { - type Item = AudioFrame; + type Item = AudioFrame<'static>; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { self.frame_rx.poll_recv(cx) @@ -73,14 +73,13 @@ impl Stream for NativeAudioStream { } pub struct AudioTrackObserver { - frame_tx: mpsc::UnboundedSender, + frame_tx: mpsc::UnboundedSender>, } impl sys_at::AudioSink for AudioTrackObserver { fn on_data(&self, data: &[i16], sample_rate: i32, nb_channels: usize, nb_frames: usize) { - // TODO(theomonnom): Should we avoid copy here? let _ = self.frame_tx.send(AudioFrame { - data: data.to_owned(), + data: data.to_owned().into(), sample_rate: sample_rate as u32, num_channels: nb_channels as u32, samples_per_channel: nb_frames as u32, diff --git a/livekit-webrtc/src/native/peer_connection.rs b/livekit-webrtc/src/native/peer_connection.rs index 3be69fb8..d7a4b136 100644 --- a/livekit-webrtc/src/native/peer_connection.rs +++ b/livekit-webrtc/src/native/peer_connection.rs @@ -392,9 +392,7 @@ impl PeerConnection { match res { Ok(sys_handle) => Ok(RtpTransceiver { - handle: imp_rt::RtpTransceiver { - sys_handle: sys_handle, - }, + handle: imp_rt::RtpTransceiver { sys_handle }, }), Err(e) => unsafe { Err(sys_err::ffi::RtcError::from(e.what()).into()) }, } diff --git a/livekit-webrtc/src/native/video_frame.rs b/livekit-webrtc/src/native/video_frame.rs index b2fb20f4..456d01b2 100644 --- a/livekit-webrtc/src/native/video_frame.rs +++ b/livekit-webrtc/src/native/video_frame.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::yuv_helper::{self, ConvertError}; +use super::yuv_helper; use crate::video_frame::VideoRotation; use crate::video_frame::{self as vf, VideoFormatType}; use cxx::UniquePtr; @@ -28,7 +28,7 @@ pub fn new_video_frame_buffer( mut sys_handle: UniquePtr, ) -> Box { unsafe { - match sys_handle.buffer_type().into() { + match sys_handle.buffer_type() { vfb_sys::ffi::VideoFrameBufferType::Native => Box::new(vf::native::NativeBuffer { handle: NativeBuffer { sys_handle }, }), @@ -184,7 +184,7 @@ impl NativeBuffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_i420() .to_argb(format, dst, dst_stride, dst_width, dst_height) } @@ -275,7 +275,7 @@ impl I420Buffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { impl_to_argb!( I420Buffer [ @@ -376,7 +376,7 @@ impl I420ABuffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_i420() .to_argb(format, dst, dst_stride, dst_width, dst_height) } @@ -470,7 +470,7 @@ impl I422Buffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_i420() .to_argb(format, dst, dst_stride, dst_width, dst_height) } @@ -556,7 +556,7 @@ impl I444Buffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_i420() .to_argb(format, dst, dst_stride, dst_width, dst_height) } @@ -644,7 +644,7 @@ impl I010Buffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_i420() .to_argb(format, dst, dst_stride, dst_width, dst_height) } @@ -756,7 +756,7 @@ impl NV12Buffer { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_i420() .to_argb(format, dst, dst_stride, dst_width, dst_height) } diff --git a/livekit-webrtc/src/native/yuv_helper.rs b/livekit-webrtc/src/native/yuv_helper.rs index e6e8f937..e14af5b1 100644 --- a/livekit-webrtc/src/native/yuv_helper.rs +++ b/livekit-webrtc/src/native/yuv_helper.rs @@ -12,34 +12,56 @@ // See the License for the specific language governing permissions and // limitations under the License. -use thiserror::Error; +#![allow(clippy::too_many_arguments)] + use webrtc_sys::yuv_helper as yuv_sys; -#[derive(Error, Debug)] -pub enum ConvertError { - #[error("conversion failed: {0}")] - Convert(&'static str), +fn argb_assert_safety(src: &[u8], src_stride: u32, _width: i32, height: i32) { + let height_abs = height.unsigned_abs(); + let min = (src_stride * height_abs) as usize; + assert!(src.len() >= min, "src isn't large enough"); } -#[inline] -fn argb_assert_safety( - src: &[u8], - src_stride: u32, +fn i420_assert_safety( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, _width: i32, height: i32, -) -> Result<(), ConvertError> { - let height_abs = height.abs() as u32; - let min = (src_stride * height_abs) as usize; +) { + let height_abs = height.unsigned_abs(); + let chroma_height = (height_abs + 1) / 2; + let min_y = (src_stride_y * height_abs) as usize; + let min_u = (src_stride_u * chroma_height) as usize; + let min_v = (src_stride_v * chroma_height) as usize; - if src.len() < min { - return Err(ConvertError::Convert("dst isn't large enough")); - } + assert!(src_y.len() >= min_y, "src_y isn't large enough"); + assert!(src_u.len() >= min_u, "src_u isn't large enough"); + assert!(src_v.len() >= min_v, "src_v isn't large enough"); +} - Ok(()) +fn nv12_assert_safety( + src_y: &[u8], + src_stride_y: u32, + src_uv: &[u8], + src_stride_uv: u32, + _width: i32, + height: i32, +) { + let height_abs = height.unsigned_abs(); + let chroma_height = (height_abs + 1) / 2; + + let min_y = (src_stride_y * height_abs) as usize; + let min_uv = (src_stride_uv * chroma_height) as usize; + + assert!(src_y.len() >= min_y, "src_y isn't large enough"); + assert!(src_uv.len() >= min_uv, "src_uv isn't large enough"); } -#[inline] -fn i420_assert_safety( +fn i444_assert_safety( src_y: &[u8], src_stride_y: u32, src_u: &[u8], @@ -48,29 +70,59 @@ fn i420_assert_safety( src_stride_v: u32, _width: i32, height: i32, -) -> Result<(), ConvertError> { - let height_abs = height.abs() as u32; - let chroma_height = (height_abs + 1) / 2; +) { + let height_abs = height.unsigned_abs(); let min_y = (src_stride_y * height_abs) as usize; - let min_u = (src_stride_u * chroma_height) as usize; - let min_v = (src_stride_v * chroma_height) as usize; + let min_u = (src_stride_u * height_abs) as usize; + let min_v = (src_stride_v * height_abs) as usize; - if src_y.len() < min_y { - return Err(ConvertError::Convert("src_y isn't large enough")); - } + assert!(src_y.len() >= min_y, "src_y isn't large enough"); + assert!(src_u.len() >= min_u, "src_u isn't large enough"); + assert!(src_v.len() >= min_v, "src_v isn't large enough"); +} - if src_u.len() < min_u { - return Err(ConvertError::Convert("src_u isn't large enough")); - } +fn i422_assert_safety( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + _width: i32, + height: i32, +) { + let height_abs = height.unsigned_abs(); + let min_y = (src_stride_y * height_abs) as usize; + let min_u = (src_stride_u * height_abs) as usize; + let min_v = (src_stride_v * height_abs) as usize; - if src_v.len() < min_v { - return Err(ConvertError::Convert("src_v isn't large enough")); - } + assert!(src_y.len() >= min_y, "src_y isn't large enough"); + assert!(src_u.len() >= min_u, "src_u isn't large enough"); + assert!(src_v.len() >= min_v, "src_v isn't large enough"); +} + +fn i010_assert_safety( + src_y: &[u16], + src_stride_y: u32, + src_u: &[u16], + src_stride_u: u32, + src_v: &[u16], + src_stride_v: u32, + _width: i32, + height: i32, +) { + let height_abs: u32 = height.unsigned_abs(); + let chroma_height = height_abs / 2; + let min_y = (src_stride_y * height_abs) as usize / 2; + let min_u = (src_stride_u * chroma_height) as usize / 2; + let min_v = (src_stride_v * chroma_height) as usize / 2; - Ok(()) + assert!(src_y.len() >= min_y, "src_y isn't large enough"); + assert!(src_u.len() >= min_u, "src_u isn't large enough"); + assert!(src_v.len() >= min_v, "src_v isn't large enough"); } -macro_rules! i420_to_x { +macro_rules! i420_to_rgba { ($x:ident) => { pub fn $x( src_y: &[u8], @@ -83,8 +135,7 @@ macro_rules! i420_to_x { dst_stride: u32, width: i32, height: i32, - ) -> Result<(), ConvertError> { - argb_assert_safety(dst, dst_stride, width, height)?; + ) { i420_assert_safety( src_y, src_stride_y, @@ -94,7 +145,8 @@ macro_rules! i420_to_x { src_stride_v, width, height, - )?; + ); + argb_assert_safety(dst, dst_stride, width, height); unsafe { yuv_sys::ffi::$x( @@ -111,13 +163,11 @@ macro_rules! i420_to_x { ) .unwrap(); } - - Ok(()) } }; } -macro_rules! x_to_i420 { +macro_rules! rgba_to_i420 { ($x:ident) => { pub fn $x( src_argb: &[u8], @@ -130,8 +180,7 @@ macro_rules! x_to_i420 { dst_stride_v: u32, width: i32, height: i32, - ) -> Result<(), ConvertError> { - argb_assert_safety(src_argb, src_stride_argb, width, height)?; + ) { i420_assert_safety( dst_y, dst_stride_y, @@ -141,7 +190,8 @@ macro_rules! x_to_i420 { dst_stride_v, width, height, - )?; + ); + argb_assert_safety(src_argb, src_stride_argb, width, height); unsafe { yuv_sys::ffi::$x( @@ -158,8 +208,6 @@ macro_rules! x_to_i420 { ) .unwrap(); } - - Ok(()) } }; } @@ -171,9 +219,9 @@ pub fn argb_to_rgb24( dst_stride_rgb24: u32, width: i32, height: i32, -) -> Result<(), ConvertError> { - argb_assert_safety(src_argb, src_stride_argb, width, height)?; - argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height)?; +) { + argb_assert_safety(src_argb, src_stride_argb, width, height); + argb_assert_safety(dst_rgb24, dst_stride_rgb24, width, height); unsafe { yuv_sys::ffi::argb_to_rgb24( @@ -186,14 +234,590 @@ pub fn argb_to_rgb24( ) .unwrap(); } +} + +// I420 <> RGB conversion +rgba_to_i420!(argb_to_i420); +rgba_to_i420!(abgr_to_i420); + +i420_to_rgba!(i420_to_argb); +i420_to_rgba!(i420_to_bgra); +i420_to_rgba!(i420_to_abgr); +i420_to_rgba!(i420_to_rgba); + +pub fn nv12_to_i420( + src_y: &[u8], + src_stride_y: u32, + src_uv: &[u8], + src_stride_uv: u32, + dst_y: &mut [u8], + dst_stride_y: u32, + dst_u: &mut [u8], + dst_stride_u: u32, + dst_v: &mut [u8], + dst_stride_v: u32, + width: i32, + height: i32, +) { + nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height); + i420_assert_safety( + dst_y, + dst_stride_y, + dst_u, + dst_stride_u, + dst_v, + dst_stride_v, + width, + height, + ); + + unsafe { + yuv_sys::ffi::nv12_to_i420( + src_y.as_ptr(), + src_stride_y as i32, + src_uv.as_ptr(), + src_stride_uv as i32, + dst_y.as_mut_ptr(), + dst_stride_y as i32, + dst_u.as_mut_ptr(), + dst_stride_u as i32, + dst_v.as_mut_ptr(), + dst_stride_v as i32, + width, + height, + ) + .unwrap(); + } +} + +pub fn i444_to_i420( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + dst_y: &mut [u8], + dst_stride_y: u32, + dst_u: &mut [u8], + dst_stride_u: u32, + dst_v: &mut [u8], + dst_stride_v: u32, + width: i32, + height: i32, +) { + i444_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + i420_assert_safety( + dst_y, + dst_stride_y, + dst_u, + dst_stride_u, + dst_v, + dst_stride_v, + width, + height, + ); + + unsafe { + yuv_sys::ffi::i444_to_i420( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_y.as_mut_ptr(), + dst_stride_y as i32, + dst_u.as_mut_ptr(), + dst_stride_u as i32, + dst_v.as_mut_ptr(), + dst_stride_v as i32, + width, + height, + ) + .unwrap(); + } +} + +pub fn i422_to_i420( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + dst_y: &mut [u8], + dst_stride_y: u32, + dst_u: &mut [u8], + dst_stride_u: u32, + dst_v: &mut [u8], + dst_stride_v: u32, + width: i32, + height: i32, +) { + i422_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + i420_assert_safety( + dst_y, + dst_stride_y, + dst_u, + dst_stride_u, + dst_v, + dst_stride_v, + width, + height, + ); + + unsafe { + yuv_sys::ffi::i422_to_i420( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_y.as_mut_ptr(), + dst_stride_y as i32, + dst_u.as_mut_ptr(), + dst_stride_u as i32, + dst_v.as_mut_ptr(), + dst_stride_v as i32, + width, + height, + ) + .unwrap() + } +} + +pub fn i010_to_i420( + src_y: &[u16], + src_stride_y: u32, + src_u: &[u16], + src_stride_u: u32, + src_v: &[u16], + src_stride_v: u32, + dst_y: &mut [u8], + dst_stride_y: u32, + dst_u: &mut [u8], + dst_stride_u: u32, + dst_v: &mut [u8], + dst_stride_v: u32, + width: i32, + height: i32, +) { + i010_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + i420_assert_safety( + dst_y, + dst_stride_y, + dst_u, + dst_stride_u, + dst_v, + dst_stride_v, + width, + height, + ); + + unsafe { + yuv_sys::ffi::i010_to_i420( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_y.as_mut_ptr(), + dst_stride_y as i32, + dst_u.as_mut_ptr(), + dst_stride_u as i32, + dst_v.as_mut_ptr(), + dst_stride_v as i32, + width, + height, + ) + .unwrap() + } +} + +pub fn nv12_to_argb( + src_y: &[u8], + src_stride_y: u32, + src_uv: &[u8], + src_stride_uv: u32, + dst_argb: &mut [u8], + dst_stride_argb: u32, + width: i32, + height: i32, +) { + nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height); + argb_assert_safety(dst_argb, dst_stride_argb, width, height); + + unsafe { + yuv_sys::ffi::nv12_to_argb( + src_y.as_ptr(), + src_stride_y as i32, + src_uv.as_ptr(), + src_stride_uv as i32, + dst_argb.as_mut_ptr(), + dst_stride_argb as i32, + width, + height, + ) + .unwrap(); + } +} + +pub fn nv12_to_abgr( + src_y: &[u8], + src_stride_y: u32, + src_uv: &[u8], + src_stride_uv: u32, + dst_abgr: &mut [u8], + dst_stride_abgr: u32, + width: i32, + height: i32, +) { + nv12_assert_safety(src_y, src_stride_y, src_uv, src_stride_uv, width, height); + argb_assert_safety(dst_abgr, dst_stride_abgr, width, height); + + unsafe { + yuv_sys::ffi::nv12_to_abgr( + src_y.as_ptr(), + src_stride_y as i32, + src_uv.as_ptr(), + src_stride_uv as i32, + dst_abgr.as_mut_ptr(), + dst_stride_abgr as i32, + width, + height, + ) + .unwrap(); + } +} + +pub fn i444_to_argb( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + dst_argb: &mut [u8], + dst_stride_argb: u32, + width: i32, + height: i32, +) { + i444_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + argb_assert_safety(dst_argb, dst_stride_argb, width, height); - Ok(()) + unsafe { + yuv_sys::ffi::i444_to_argb( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_argb.as_mut_ptr(), + dst_stride_argb as i32, + width, + height, + ) + .unwrap(); + } } -x_to_i420!(argb_to_i420); -x_to_i420!(abgr_to_i420); +pub fn i444_to_abgr( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + dst_abgr: &mut [u8], + dst_stride_abgr: u32, + width: i32, + height: i32, +) { + i444_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + argb_assert_safety(dst_abgr, dst_stride_abgr, width, height); + + unsafe { + yuv_sys::ffi::i444_to_abgr( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_abgr.as_mut_ptr(), + dst_stride_abgr as i32, + width, + height, + ) + .unwrap() + } +} -i420_to_x!(i420_to_argb); -i420_to_x!(i420_to_bgra); -i420_to_x!(i420_to_abgr); -i420_to_x!(i420_to_rgba); +pub fn i422_to_argb( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + dst_argb: &mut [u8], + dst_stride_argb: u32, + width: i32, + height: i32, +) { + i422_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + argb_assert_safety(dst_argb, dst_stride_argb, width, height); + + unsafe { + yuv_sys::ffi::i422_to_argb( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_argb.as_mut_ptr(), + dst_stride_argb as i32, + width, + height, + ) + .unwrap(); + } +} + +pub fn i422_to_abgr( + src_y: &[u8], + src_stride_y: u32, + src_u: &[u8], + src_stride_u: u32, + src_v: &[u8], + src_stride_v: u32, + dst_abgr: &mut [u8], + dst_stride_abgr: u32, + width: i32, + height: i32, +) { + i422_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + argb_assert_safety(dst_abgr, dst_stride_abgr, width, height); + + unsafe { + yuv_sys::ffi::i422_to_abgr( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_abgr.as_mut_ptr(), + dst_stride_abgr as i32, + width, + height, + ) + .unwrap() + } +} + +pub fn i010_to_argb( + src_y: &[u16], + src_stride_y: u32, + src_u: &[u16], + src_stride_u: u32, + src_v: &[u16], + src_stride_v: u32, + dst_argb: &mut [u8], + dst_stride_argb: u32, + width: i32, + height: i32, +) { + i010_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + argb_assert_safety(dst_argb, dst_stride_argb, width, height); + + unsafe { + yuv_sys::ffi::i010_to_argb( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_argb.as_mut_ptr(), + dst_stride_argb as i32, + width, + height, + ) + .unwrap() + } +} + +pub fn i010_to_abgr( + src_y: &[u16], + src_stride_y: u32, + src_u: &[u16], + src_stride_u: u32, + src_v: &[u16], + src_stride_v: u32, + dst_abgr: &mut [u8], + dst_stride_abgr: u32, + width: i32, + height: i32, +) { + i010_assert_safety( + src_y, + src_stride_y, + src_u, + src_stride_u, + src_v, + src_stride_v, + width, + height, + ); + argb_assert_safety(dst_abgr, dst_stride_abgr, width, height); + + unsafe { + yuv_sys::ffi::i010_to_abgr( + src_y.as_ptr(), + src_stride_y as i32, + src_u.as_ptr(), + src_stride_u as i32, + src_v.as_ptr(), + src_stride_v as i32, + dst_abgr.as_mut_ptr(), + dst_stride_abgr as i32, + width, + height, + ) + .unwrap() + } +} + +pub fn abgr_to_nv12( + src_abgr: &[u8], + src_stride_abgr: u32, + dst_y: &mut [u8], + dst_stride_y: u32, + dst_uv: &mut [u8], + dst_stride_uv: u32, + width: i32, + height: i32, +) { + argb_assert_safety(src_abgr, src_stride_abgr, width, height); + nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height); + + unsafe { + yuv_sys::ffi::abgr_to_nv12( + src_abgr.as_ptr(), + src_stride_abgr as i32, + dst_y.as_mut_ptr(), + dst_stride_y as i32, + dst_uv.as_mut_ptr(), + dst_stride_uv as i32, + width, + height, + ) + .unwrap() + } +} + +pub fn argb_to_nv12( + src_argb: &[u8], + src_stride_argb: u32, + dst_y: &mut [u8], + dst_stride_y: u32, + dst_uv: &mut [u8], + dst_stride_uv: u32, + width: i32, + height: i32, +) { + argb_assert_safety(src_argb, src_stride_argb, width, height); + nv12_assert_safety(dst_y, dst_stride_y, dst_uv, dst_stride_uv, width, height); + + unsafe { + yuv_sys::ffi::argb_to_nv12( + src_argb.as_ptr(), + src_stride_argb as i32, + dst_y.as_mut_ptr(), + dst_stride_y as i32, + dst_uv.as_mut_ptr(), + dst_stride_uv as i32, + width, + height, + ) + .unwrap() + } +} diff --git a/livekit-webrtc/src/video_frame.rs b/livekit-webrtc/src/video_frame.rs index 5ad48b74..93ad6ad1 100644 --- a/livekit-webrtc/src/video_frame.rs +++ b/livekit-webrtc/src/video_frame.rs @@ -48,7 +48,6 @@ pub enum VideoFrameBufferType { I444, I010, NV12, - WebGl, } #[derive(Debug)] @@ -82,7 +81,7 @@ pub(crate) mod internal { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), super::native::ConvertError>; + ); } } @@ -148,7 +147,7 @@ macro_rules! new_buffer_type { stride: u32, width: i32, height: i32, - ) -> Result<(), $crate::video_frame::native::ConvertError> { + ) { self.handle.to_argb(format, dst, stride, width, height) } } @@ -397,8 +396,6 @@ pub mod native { use super::{vf_imp, I420Buffer, VideoFormatType, VideoFrameBuffer, VideoFrameBufferType}; use std::fmt::Debug; - pub use crate::imp::yuv_helper::ConvertError; - new_buffer_type!(NativeBuffer, Native, as_native); pub trait I420BufferExt { @@ -420,7 +417,7 @@ pub mod native { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError>; + ); } impl VideoFrameBufferExt for T { @@ -435,7 +432,7 @@ pub mod native { dst_stride: u32, dst_width: i32, dst_height: i32, - ) -> Result<(), ConvertError> { + ) { self.to_argb(format, dst, dst_stride, dst_width, dst_height) } } diff --git a/webrtc-sys/build/src/lib.rs b/webrtc-sys/build/src/lib.rs index 5613deab..d938c4bd 100644 --- a/webrtc-sys/build/src/lib.rs +++ b/webrtc-sys/build/src/lib.rs @@ -26,7 +26,7 @@ use regex::Regex; use reqwest::StatusCode; pub const SCRATH_PATH: &str = "livekit_webrtc"; -pub const WEBRTC_TAG: &str = "webrtc-b0f3927"; +pub const WEBRTC_TAG: &str = "webrtc-fcdef0d"; pub const IGNORE_DEFINES: [&str; 2] = ["CR_CLANG_REVISION", "CR_XCODE_VERSION"]; pub fn target_os() -> String { diff --git a/webrtc-sys/include/livekit/yuv_helper.h b/webrtc-sys/include/livekit/yuv_helper.h index 173c8656..acec2673 100644 --- a/webrtc-sys/include/livekit/yuv_helper.h +++ b/webrtc-sys/include/livekit/yuv_helper.h @@ -130,4 +130,235 @@ static void argb_to_rgb24(const uint8_t* src_argb, dst_stride_rgb24, width, height)); } +static void i420_to_nv12(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height) { + THROW_ON_ERROR(webrtc::I420ToNV12(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_y, dst_stride_y, + dst_uv, dst_stride_uv, width, height)); +} + +static void nv12_to_i420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + THROW_ON_ERROR(webrtc::NV12ToI420(src_y, src_stride_y, src_uv, src_stride_uv, + dst_y, dst_stride_y, dst_u, dst_stride_u, + dst_v, dst_stride_v, width, height)); +} + +static void i444_to_i420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + THROW_ON_ERROR(webrtc::I444ToI420( + src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, width, height)); +} + +static void i422_to_i420(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + THROW_ON_ERROR(webrtc::I422ToI420( + src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, width, height)); +} + +static void i010_to_i420(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_u, + int dst_stride_u, + uint8_t* dst_v, + int dst_stride_v, + int width, + int height) { + THROW_ON_ERROR(webrtc::I010ToI420( + src_y, src_stride_y, src_u, src_stride_u, src_v, src_stride_v, dst_y, + dst_stride_y, dst_u, dst_stride_u, dst_v, dst_stride_v, width, height)); +} + +static void nv12_to_argb(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + THROW_ON_ERROR(webrtc::NV12ToARGB(src_y, src_stride_y, src_uv, src_stride_uv, + dst_argb, dst_stride_argb, width, height)); +} + +static void nv12_to_abgr(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_uv, + int src_stride_uv, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + THROW_ON_ERROR(webrtc::NV12ToABGR(src_y, src_stride_y, src_uv, src_stride_uv, + dst_abgr, dst_stride_abgr, width, height)); +} + +static void i444_to_argb(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + THROW_ON_ERROR(webrtc::I444ToARGB(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_abgr, + dst_stride_abgr, width, height)); +} + +static void i444_to_abgr(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + THROW_ON_ERROR(webrtc::I444ToABGR(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_abgr, + dst_stride_abgr, width, height)); +} + +static void i422_to_argb(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + THROW_ON_ERROR(webrtc::I422ToARGB(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_argb, + dst_stride_argb, width, height)); +} + +static void i422_to_abgr(const uint8_t* src_y, + int src_stride_y, + const uint8_t* src_u, + int src_stride_u, + const uint8_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + THROW_ON_ERROR(webrtc::I422ToABGR(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_abgr, + dst_stride_abgr, width, height)); +} + +static void i010_to_argb(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_argb, + int dst_stride_argb, + int width, + int height) { + THROW_ON_ERROR(webrtc::I010ToARGB(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_argb, + dst_stride_argb, width, height)); +} + +static void i010_to_abgr(const uint16_t* src_y, + int src_stride_y, + const uint16_t* src_u, + int src_stride_u, + const uint16_t* src_v, + int src_stride_v, + uint8_t* dst_abgr, + int dst_stride_abgr, + int width, + int height) { + THROW_ON_ERROR(webrtc::I010ToABGR(src_y, src_stride_y, src_u, src_stride_u, + src_v, src_stride_v, dst_abgr, + dst_stride_abgr, width, height)); +} + +static void abgr_to_nv12(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height) { + THROW_ON_ERROR(webrtc::ABGRToNV12(src_abgr, src_stride_abgr, dst_y, + dst_stride_y, dst_uv, dst_stride_uv, width, + height)); +} + +static void argb_to_nv12(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_y, + int dst_stride_y, + uint8_t* dst_uv, + int dst_stride_uv, + int width, + int height) { + THROW_ON_ERROR(webrtc::ARGBToNV12(src_argb, src_stride_argb, dst_y, + dst_stride_y, dst_uv, dst_stride_uv, width, + height)); +} + } // namespace livekit diff --git a/webrtc-sys/src/yuv_helper.rs b/webrtc-sys/src/yuv_helper.rs index fe55573a..c1ddb5c6 100644 --- a/webrtc-sys/src/yuv_helper.rs +++ b/webrtc-sys/src/yuv_helper.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[allow(clippy::too_many_arguments)] +#[allow(clippy::missing_safety_doc)] #[cxx::bridge(namespace = "livekit")] pub mod ffi { unsafe extern "C++" { @@ -103,5 +105,193 @@ pub mod ffi { width: i32, height: i32, ) -> Result<()>; + + unsafe fn nv12_to_i420( + src_y: *const u8, + src_stride_y: i32, + src_uv: *const u8, + src_stride_uv: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_u: *mut u8, + dst_stride_u: i32, + dst_v: *mut u8, + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i444_to_i420( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_u: *mut u8, + dst_stride_u: i32, + dst_v: *mut u8, + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i422_to_i420( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_u: *mut u8, + dst_stride_u: i32, + dst_v: *mut u8, + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i010_to_i420( + src_y: *const u16, + src_stride_y: i32, + src_u: *const u16, + src_stride_u: i32, + src_v: *const u16, + src_stride_v: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_u: *mut u8, + dst_stride_u: i32, + dst_v: *mut u8, + dst_stride_v: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn nv12_to_argb( + src_y: *const u8, + src_stride_y: i32, + src_uv: *const u8, + src_stride_uv: i32, + dst_argb: *mut u8, + dst_stride_argb: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn nv12_to_abgr( + src_y: *const u8, + src_stride_y: i32, + src_uv: *const u8, + src_stride_uv: i32, + dst_abgr: *mut u8, + dst_stride_abgr: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i444_to_argb( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_abgr: *mut u8, + dst_stride_abgr: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i444_to_abgr( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_abgr: *mut u8, + dst_stride_abgr: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i422_to_argb( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_argb: *mut u8, + dst_stride_argb: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i422_to_abgr( + src_y: *const u8, + src_stride_y: i32, + src_u: *const u8, + src_stride_u: i32, + src_v: *const u8, + src_stride_v: i32, + dst_abgr: *mut u8, + dst_stride_abgr: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i010_to_argb( + src_y: *const u16, + src_stride_y: i32, + src_u: *const u16, + src_stride_u: i32, + src_v: *const u16, + src_stride_v: i32, + dst_argb: *mut u8, + dst_stride_argb: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn i010_to_abgr( + src_y: *const u16, + src_stride_y: i32, + src_u: *const u16, + src_stride_u: i32, + src_v: *const u16, + src_stride_v: i32, + dst_abgr: *mut u8, + dst_stride_abgr: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn abgr_to_nv12( + src_abgr: *const u8, + src_stride_abgr: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_uv: *mut u8, + dst_stride_uv: i32, + width: i32, + height: i32, + ) -> Result<()>; + + unsafe fn argb_to_nv12( + src_argb: *const u8, + src_stride_argb: i32, + dst_y: *mut u8, + dst_stride_y: i32, + dst_uv: *mut u8, + dst_stride_uv: i32, + width: i32, + height: i32, + ) -> Result<()>; } }