From 2df1040e1ecdeb251a08586883ab3b4c2540c723 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Thu, 24 Aug 2023 21:00:23 -0700 Subject: [PATCH 01/20] send it --- go.mod | 18 +-- go.sum | 43 +++--- pkg/config/pipeline.go | 37 +++++ pkg/config/service.go | 5 + pkg/pipeline/builder/audio.go | 35 +++-- pkg/pipeline/builder/video.go | 120 +++++++-------- pkg/pipeline/controller.go | 34 +++-- pkg/pipeline/source/sdk.go | 84 ++++++++++- pkg/pipeline/source/source.go | 3 +- pkg/stats/monitor.go | 12 ++ pkg/types/types.go | 1 + test/participant.go | 274 ++++++++++++++++++++++++++++++++++ test/runner.go | 1 + 13 files changed, 541 insertions(+), 126 deletions(-) create mode 100644 test/participant.go diff --git a/go.mod b/go.mod index 28535e8c..1b4dc365 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/livekit/livekit-server v1.4.5-0.20230814182001-77c8e824735b github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/protocol v1.6.0 + github.com/livekit/protocol v1.6.1-0.20230816204336-79e783089699 github.com/livekit/psrpc v0.3.2 github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8 github.com/pion/rtp v1.8.1 @@ -31,7 +31,7 @@ require ( github.com/tinyzimmer/go-gst v0.2.33 github.com/urfave/cli/v2 v2.25.7 go.uber.org/atomic v1.11.0 - go.uber.org/zap v1.24.0 + go.uber.org/zap v1.25.0 google.golang.org/api v0.130.0 google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 @@ -44,7 +44,6 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.0 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect - github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bep/debounce v1.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -108,15 +107,14 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - go.uber.org/goleak v1.1.12 // indirect - go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.11.0 // indirect - golang.org/x/exp v0.0.0-20230810033253-352e893a4cad // indirect - golang.org/x/net v0.13.0 // indirect + go.uber.org/multierr v1.10.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.9.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index d0281fc6..a1261f0f 100644 --- a/go.sum +++ b/go.sum @@ -32,7 +32,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/aws/aws-sdk-go v1.44.296 h1:ALRZIIKI+6EBWDiWP4RHWmOtHZ7dywRzenL4NWgNI2A= github.com/aws/aws-sdk-go v1.44.296/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= @@ -177,8 +176,8 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08 h1:e0qwVjrtzmADgNZpdgSJgyhlF6BgrHkpdnkONL8pLrw= github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14= -github.com/livekit/protocol v1.6.0 h1:19S+vFZqnivKIOpyR3DEK/mSaykQ3UEf7H2G/mBOE54= -github.com/livekit/protocol v1.6.0/go.mod h1:SUS9foM1xBzw/AFrgTJuFX/oSuwlnIbHmpdiPdCvwEM= +github.com/livekit/protocol v1.6.1-0.20230816204336-79e783089699 h1:8EsXCLF16x00RhpC8rFB/lo9STMj8qWlcknSh+kOVeA= +github.com/livekit/protocol v1.6.1-0.20230816204336-79e783089699/go.mod h1:KBa63PkgSJOAA+qPf22FRzouVYjKIBKfCkFyX04DqVU= github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8 h1:Vm64l3U2rWAB2IdG/0/CXIZTgVPmldsQrDAOcq+zQQs= @@ -280,7 +279,6 @@ github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -301,7 +299,6 @@ github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6S github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -309,15 +306,13 @@ github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaD go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -328,17 +323,16 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= +golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -356,7 +350,6 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -369,8 +362,9 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= @@ -380,7 +374,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= @@ -398,9 +391,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201207223542-d4d67f95c62d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -412,8 +403,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -436,8 +428,9 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -447,7 +440,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -496,7 +488,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/pkg/config/pipeline.go b/pkg/config/pipeline.go index 31585135..639de545 100644 --- a/pkg/config/pipeline.go +++ b/pkg/config/pipeline.go @@ -290,6 +290,43 @@ func (p *PipelineConfig) Update(request *rpc.StartEgressRequest) error { return err } + case *rpc.StartEgressRequest_Participant: + p.RequestType = types.RequestTypeParticipant + clone := proto.Clone(req.Participant).(*livekit.ParticipantEgressRequest) + p.Info.Request = &livekit.EgressInfo_Participant{ + Participant: clone, + } + redactEncodedOutputs(clone) + + p.SourceType = types.SourceTypeSDK + p.Latency = sdkLatency + + p.Info.RoomName = req.Participant.RoomName + p.AudioEnabled = true + p.AudioTranscoding = true + p.VideoEnabled = true + p.VideoTranscoding = true + p.Identity = req.Participant.Identity + if p.Identity == "" { + return errors.ErrInvalidInput("identity") + } + + // encoding options + switch opts := req.Participant.Options.(type) { + case *livekit.ParticipantEgressRequest_Preset: + p.applyPreset(opts.Preset) + + case *livekit.ParticipantEgressRequest_Advanced: + if err := p.applyAdvanced(opts.Advanced); err != nil { + return err + } + } + + // output params + if err := p.updateEncodedOutputs(req.Participant); err != nil { + return err + } + case *rpc.StartEgressRequest_TrackComposite: p.RequestType = types.RequestTypeTrackComposite clone := proto.Clone(req.TrackComposite).(*livekit.TrackCompositeEgressRequest) diff --git a/pkg/config/service.go b/pkg/config/service.go index 604493c2..29b65795 100644 --- a/pkg/config/service.go +++ b/pkg/config/service.go @@ -30,6 +30,7 @@ const ( roomCompositeCpuCost = 4 webCpuCost = 4 + participantCpuCost = 1 trackCompositeCpuCost = 1 trackCpuCost = 0.5 @@ -51,6 +52,7 @@ type ServiceConfig struct { type CPUCostConfig struct { RoomCompositeCpuCost float64 `yaml:"room_composite_cpu_cost"` WebCpuCost float64 `yaml:"web_cpu_cost"` + ParticipantCpuCost float64 `yaml:"participant_cpu_cost"` TrackCompositeCpuCost float64 `yaml:"track_composite_cpu_cost"` TrackCpuCost float64 `yaml:"track_cpu_cost"` } @@ -83,6 +85,9 @@ func NewServiceConfig(confString string) (*ServiceConfig, error) { if conf.WebCpuCost <= 0 { conf.WebCpuCost = webCpuCost } + if conf.ParticipantCpuCost <= 0 { + conf.ParticipantCpuCost = participantCpuCost + } if conf.TrackCompositeCpuCost <= 0 { conf.TrackCompositeCpuCost = trackCompositeCpuCost } diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index 2198cc2f..3586d905 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -53,6 +53,10 @@ func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*gst } } + if err := pipeline.AddSourceBin(b); err != nil { + return nil, err + } + return b, nil } @@ -83,7 +87,7 @@ func buildWebAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { func buildSDKAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { if p.AudioTrack != nil { - if err := buildAudioAppSrcBin(b, p); err != nil { + if err := AddAudioAppSrcBin(b, p, p.AudioTrack); err != nil { return err } } @@ -102,28 +106,23 @@ func buildSDKAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { return nil } -func buildAudioAppSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig) error { - track := p.AudioTrack - - b := audioBin.NewBin(track.TrackID) - b.SetEOSFunc(track.EOSFunc) - if err := audioBin.AddSourceBin(b); err != nil { - return err - } +func AddAudioAppSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) error { + b := audioBin.NewBin(ts.TrackID) + b.SetEOSFunc(ts.EOSFunc) - track.AppSrc.Element.SetArg("format", "time") - if err := track.AppSrc.Element.SetProperty("is-live", true); err != nil { + ts.AppSrc.Element.SetArg("format", "time") + if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { return err } - if err := b.AddElement(track.AppSrc.Element); err != nil { + if err := b.AddElement(ts.AppSrc.Element); err != nil { return err } - switch track.MimeType { + switch ts.MimeType { case types.MimeTypeOpus: - if err := track.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( + if err := ts.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( "application/x-rtp,media=audio,payload=%d,encoding-name=OPUS,clock-rate=%d", - track.PayloadType, track.ClockRate, + ts.PayloadType, ts.ClockRate, ))); err != nil { return errors.ErrGstPipelineError(err) } @@ -143,13 +142,17 @@ func buildAudioAppSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig) erro } default: - return errors.ErrNotSupported(string(track.MimeType)) + return errors.ErrNotSupported(string(ts.MimeType)) } if err := addAudioConverter(b, p); err != nil { return err } + if err := audioBin.AddSourceBin(b); err != nil { + return err + } + return nil } diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index 0eca1e85..a9f6670d 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -33,33 +33,50 @@ import ( const videoTestSrcName = "video_test_src" -func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*gstreamer.Bin, error) { +type VideoInput struct { + lastPTS atomic.Duration + nextPTS atomic.Duration + selectedPad string + nextPad string + + mu sync.Mutex + pads map[string]*gst.Pad + selector *gst.Element +} + +func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*VideoInput, *gstreamer.Bin, error) { + v := &VideoInput{} + b := pipeline.NewBin("video") switch p.SourceType { case types.SourceTypeSDK: - if err := buildSDKVideoInput(b, p); err != nil { - return nil, err + if err := v.buildSDKVideoInput(b, p); err != nil { + return nil, nil, err } case types.SourceTypeWeb: if err := buildWebVideoInput(b, p); err != nil { - return nil, err + return nil, nil, err } } if len(p.Outputs) > 1 { tee, err := gst.NewElementWithName("tee", "video_tee") if err != nil { - return nil, err + return nil, nil, err } if err = b.AddElement(tee); err != nil { - return nil, err + return nil, nil, err } } - return b, nil + if err := pipeline.AddSourceBin(b); err != nil { + return nil, nil, err + } + + return v, b, nil } func buildWebVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { @@ -111,24 +128,18 @@ func buildWebVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { return nil } -type videoSDKBin struct { - lastPTS atomic.Duration - nextPTS atomic.Duration - selectedPad string - nextPad string +func (v *VideoInput) buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { + v.pads = make(map[string]*gst.Pad) - mu sync.Mutex - pads map[string]*gst.Pad - selector *gst.Element -} - -func buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { - v := &videoSDKBin{ - pads: make(map[string]*gst.Pad), + // add selector first so pads can be created + if p.VideoTranscoding { + if err := v.addVideoSelector(b, p); err != nil { + return err + } } if p.VideoTrack != nil { - if err := v.buildVideoAppSrcBin(b, p); err != nil { + if err := v.AddVideoAppSrcBin(b, p, p.VideoTrack); err != nil { return err } } @@ -137,17 +148,8 @@ func buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { if err := v.buildVideoTestSrcBin(b, p); err != nil { return err } - if err := v.addVideoSelector(b, p); err != nil { - return err - } - v.createTestSrcPad() - if p.VideoTrack != nil { - v.createSrcPad(p.VideoTrack.TrackID) - if err := v.setSelectorPad(p.VideoTrack.TrackID); err != nil { - return err - } - } else { + if p.VideoTrack == nil { if err := v.setSelectorPad(videoTestSrcName); err != nil { return err } @@ -165,28 +167,23 @@ func buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { return nil } -func (v *videoSDKBin) buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig) error { - track := p.VideoTrack +func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) error { + b := videoBin.NewBin(ts.TrackID) + b.SetEOSFunc(ts.EOSFunc) - b := videoBin.NewBin(track.TrackID) - b.SetEOSFunc(track.EOSFunc) - if err := videoBin.AddSourceBin(b); err != nil { - return err - } - - track.AppSrc.Element.SetArg("format", "time") - if err := track.AppSrc.Element.SetProperty("is-live", true); err != nil { + ts.AppSrc.Element.SetArg("format", "time") + if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { return errors.ErrGstPipelineError(err) } - if err := b.AddElement(track.AppSrc.Element); err != nil { + if err := b.AddElement(ts.AppSrc.Element); err != nil { return err } - switch track.MimeType { + switch ts.MimeType { case types.MimeTypeH264: - if err := track.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( + if err := ts.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( "application/x-rtp,media=video,payload=%d,encoding-name=H264,clock-rate=%d", - track.PayloadType, track.ClockRate, + ts.PayloadType, ts.ClockRate, ))); err != nil { return errors.ErrGstPipelineError(err) } @@ -233,9 +230,9 @@ func (v *videoSDKBin) buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pip } case types.MimeTypeVP8: - if err := track.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( + if err := ts.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( "application/x-rtp,media=video,payload=%d,encoding-name=VP8,clock-rate=%d", - track.PayloadType, track.ClockRate, + ts.PayloadType, ts.ClockRate, ))); err != nil { return errors.ErrGstPipelineError(err) } @@ -261,9 +258,9 @@ func (v *videoSDKBin) buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pip } case types.MimeTypeVP9: - if err := track.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( + if err := ts.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( "application/x-rtp,media=video,payload=%d,encoding-name=VP9,clock-rate=%d", - track.PayloadType, track.ClockRate, + ts.PayloadType, ts.ClockRate, ))); err != nil { return errors.ErrGstPipelineError(err) } @@ -307,7 +304,7 @@ func (v *videoSDKBin) buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pip } default: - return errors.ErrNotSupported(string(track.MimeType)) + return errors.ErrNotSupported(string(ts.MimeType)) } videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) @@ -342,10 +339,14 @@ func (v *videoSDKBin) buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pip return err } - return nil + v.createSrcPad(ts.TrackID) + if err = videoBin.AddSourceBin(b); err != nil { + return err + } + return v.setSelectorPad(ts.TrackID) } -func (v *videoSDKBin) buildVideoTestSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig) error { +func (v *VideoInput) buildVideoTestSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig) error { b := videoBin.NewBin(videoTestSrcName) if err := videoBin.AddSourceBin(b); err != nil { return err @@ -369,10 +370,11 @@ func (v *videoSDKBin) buildVideoTestSrcBin(videoBin *gstreamer.Bin, p *config.Pi return err } + v.createTestSrcPad() return nil } -func (v *videoSDKBin) addVideoSelector(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (v *VideoInput) addVideoSelector(b *gstreamer.Bin, p *config.PipelineConfig) error { inputSelector, err := gst.NewElement("input-selector") if err != nil { return errors.ErrGstPipelineError(err) @@ -477,14 +479,14 @@ func addVideoEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { } } -func (v *videoSDKBin) getSrcPad(name string) *gst.Pad { +func (v *VideoInput) getSrcPad(name string) *gst.Pad { v.mu.Lock() defer v.mu.Unlock() return v.pads[name] } -func (v *videoSDKBin) createSrcPad(trackID string) { +func (v *VideoInput) createSrcPad(trackID string) { v.mu.Lock() defer v.mu.Unlock() @@ -503,7 +505,7 @@ func (v *videoSDKBin) createSrcPad(trackID string) { v.pads[trackID] = pad } -func (v *videoSDKBin) createTestSrcPad() { +func (v *VideoInput) createTestSrcPad() { v.mu.Lock() defer v.mu.Unlock() @@ -527,7 +529,7 @@ func (v *videoSDKBin) createTestSrcPad() { v.pads[videoTestSrcName] = pad } -func (v *videoSDKBin) onTrackMuted(trackID string) { +func (v *VideoInput) onTrackMuted(trackID string) { if v.selectedPad != trackID { return } @@ -537,7 +539,7 @@ func (v *videoSDKBin) onTrackMuted(trackID string) { } } -func (v *videoSDKBin) onTrackUnmuted(trackID string, pts time.Duration) { +func (v *VideoInput) onTrackUnmuted(trackID string, pts time.Duration) { v.mu.Lock() defer v.mu.Unlock() @@ -550,7 +552,7 @@ func (v *videoSDKBin) onTrackUnmuted(trackID string, pts time.Duration) { } // TODO: go-gst should accept objects directly and handle conversion to C -func (v *videoSDKBin) setSelectorPad(name string) error { +func (v *VideoInput) setSelectorPad(name string) error { v.mu.Lock() defer v.mu.Unlock() diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 618c4899..44cbf20b 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -34,6 +34,7 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/tracer" "github.com/livekit/protocol/utils" + lksdk "github.com/livekit/server-sdk-go" ) const ( @@ -119,26 +120,41 @@ func (c *Controller) BuildPipeline() error { p.SetWatch(c.messageWatch) p.AddOnStop(c.OnStop) + var audioBin, videoBin *gstreamer.Bin + var videoInput *builder.VideoInput + if c.AudioEnabled { - audioBin, err := builder.BuildAudioBin(p, c.PipelineConfig) + audioBin, err = builder.BuildAudioBin(p, c.PipelineConfig) if err != nil { return err } - if err = p.AddSourceBin(audioBin); err != nil { - return err - } } - if c.VideoEnabled { - videoBin, err := builder.BuildVideoBin(p, c.PipelineConfig) + videoInput, videoBin, err = builder.BuildVideoBin(p, c.PipelineConfig) if err != nil { return err } - if err = p.AddSourceBin(videoBin); err != nil { - return err - } } + p.AddOnTrackAdded(func(ts *config.TrackSource) { + switch ts.Kind { + case lksdk.TrackKindAudio: + if err := builder.AddAudioAppSrcBin(audioBin, c.PipelineConfig, ts); err != nil { + p.OnError(err) + } + case lksdk.TrackKindVideo: + if err := videoInput.AddVideoAppSrcBin(videoBin, c.PipelineConfig, ts); err != nil { + p.OnError(err) + } + } + }) + p.AddOnTrackRemoved(func(trackID string) { + _, err := p.RemoveSinkBin(trackID) + if err != nil { + p.OnError(err) + } + }) + for egressType := range c.Outputs { var sinkBin *gstreamer.Bin switch egressType { diff --git a/pkg/pipeline/source/sdk.go b/pkg/pipeline/source/sdk.go index 9eb18bf0..9760add5 100644 --- a/pkg/pipeline/source/sdk.go +++ b/pkg/pipeline/source/sdk.go @@ -33,6 +33,7 @@ import ( "github.com/livekit/egress/pkg/gstreamer" "github.com/livekit/egress/pkg/pipeline/source/sdk" "github.com/livekit/egress/pkg/types" + "github.com/livekit/protocol/livekit" "github.com/livekit/protocol/logger" "github.com/livekit/protocol/tracer" lksdk "github.com/livekit/server-sdk-go" @@ -144,6 +145,10 @@ func (s *SDKSource) joinRoom() error { }, OnDisconnected: s.onDisconnected, } + if s.RequestType == types.RequestTypeParticipant { + cb.ParticipantCallback.OnTrackPublished = s.onTrackPublished + cb.OnParticipantDisconnected = s.onParticipantDisconnected + } logger.Debugw("connecting to room") s.room = lksdk.CreateRoom(cb) @@ -154,9 +159,12 @@ func (s *SDKSource) joinRoom() error { var fileIdentifier string var err error switch s.RequestType { + case types.RequestTypeParticipant: + fileIdentifier = s.Identity + err = s.awaitParticipant(s.Identity) + case types.RequestTypeTrackComposite: fileIdentifier = s.Info.RoomName - tracks := make(map[string]struct{}) if s.AudioEnabled { tracks[s.AudioTrackID] = struct{}{} @@ -168,7 +176,6 @@ func (s *SDKSource) joinRoom() error { case types.RequestTypeTrack: fileIdentifier = s.TrackID - err = s.awaitTracks(map[string]struct{}{s.TrackID: {}}) } if err != nil { @@ -183,6 +190,39 @@ func (s *SDKSource) joinRoom() error { return nil } +func (s *SDKSource) awaitParticipant(identity string) error { + s.errors = make(chan error, 2) + + rp, err := s.getParticipant(identity) + if err != nil { + return err + } + + trackCount := 0 + for trackCount == 0 || trackCount < len(rp.Tracks()) { + if err = <-s.errors; err != nil { + return err + } + trackCount++ + } + + s.initialized.Break() + return nil +} + +func (s *SDKSource) getParticipant(identity string) (*lksdk.RemoteParticipant, error) { + deadline := time.Now().Add(subscriptionTimeout) + for time.Now().Before(deadline) { + for _, p := range s.room.GetParticipants() { + if p.Identity() == identity { + return p, nil + } + } + time.Sleep(100 * time.Millisecond) + } + return nil, errors.ErrParticipantNotFound(identity) +} + func (s *SDKSource) awaitTracks(expecting map[string]struct{}) error { trackCount := len(expecting) s.errors = make(chan error, trackCount) @@ -296,7 +336,11 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo ts.EOSFunc = s.CloseWriters s.audioWriter = writer - s.AudioTrack = ts + if s.initialized.IsBroken() { + s.callbacks.OnTrackAdded(ts) + } else { + s.AudioTrack = ts + } case types.MimeTypeH264, types.MimeTypeVP8, types.MimeTypeVP9: s.VideoEnabled = true @@ -316,7 +360,11 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo ts.EOSFunc = s.CloseWriters s.videoWriter = writer - s.VideoTrack = ts + if s.initialized.IsBroken() { + s.callbacks.OnTrackAdded(ts) + } else { + s.VideoTrack = ts + } default: onSubscribeErr = errors.ErrNotSupported(string(ts.MimeType)) @@ -380,6 +428,23 @@ func (s *SDKSource) createWriter( return writer, nil } +func (s *SDKSource) onTrackPublished(pub *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { + if rp.Identity() != s.Identity { + return + } + + switch pub.Source() { + case livekit.TrackSource_CAMERA, livekit.TrackSource_MICROPHONE: + if err := s.subscribe(pub); err != nil { + logger.Errorw("failed to subscribe to track", err, "trackID", pub.SID()) + } + default: + logger.Infow("ignoring participant track", + "reason", fmt.Sprintf("source %s", pub.Source())) + return + } +} + func (s *SDKSource) onTrackMuted(pub lksdk.TrackPublication, _ lksdk.Participant) { if w := s.getWriterForTrack(pub.SID()); w != nil { w.SetTrackMuted(true) @@ -421,7 +486,16 @@ func (s *SDKSource) onTrackFinished(trackID string) { } w.Drain(true) - if s.active.Dec() == 0 { + active := s.active.Dec() + if s.RequestType == types.RequestTypeParticipant { + s.callbacks.OnTrackRemoved(trackID) + } else if active == 0 { + s.onDisconnected() + } +} + +func (s *SDKSource) onParticipantDisconnected(rp *lksdk.RemoteParticipant) { + if rp.Identity() == s.Identity { s.onDisconnected() } } diff --git a/pkg/pipeline/source/source.go b/pkg/pipeline/source/source.go index a0b7aed9..b4176bf1 100644 --- a/pkg/pipeline/source/source.go +++ b/pkg/pipeline/source/source.go @@ -36,7 +36,8 @@ func New(ctx context.Context, p *config.PipelineConfig, callbacks *gstreamer.Cal types.RequestTypeWeb: return NewWebSource(ctx, p) - case types.RequestTypeTrackComposite, + case types.RequestTypeParticipant, + types.RequestTypeTrackComposite, types.RequestTypeTrack: return NewSDKSource(ctx, p, callbacks) diff --git a/pkg/stats/monitor.go b/pkg/stats/monitor.go index 64198e67..d9f72dcf 100644 --- a/pkg/stats/monitor.go +++ b/pkg/stats/monitor.go @@ -99,6 +99,7 @@ func (m *Monitor) checkCPUConfig() error { requirements := []float64{ m.cpuCostConfig.RoomCompositeCpuCost, m.cpuCostConfig.WebCpuCost, + m.cpuCostConfig.ParticipantCpuCost, m.cpuCostConfig.TrackCompositeCpuCost, m.cpuCostConfig.TrackCpuCost, } @@ -165,6 +166,8 @@ func (m *Monitor) canAcceptRequest(req *rpc.StartEgressRequest) bool { accept = available >= m.cpuCostConfig.RoomCompositeCpuCost case *rpc.StartEgressRequest_Web: accept = available >= m.cpuCostConfig.WebCpuCost + case *rpc.StartEgressRequest_Participant: + accept = available >= m.cpuCostConfig.ParticipantCpuCost case *rpc.StartEgressRequest_TrackComposite: accept = available >= m.cpuCostConfig.TrackCompositeCpuCost case *rpc.StartEgressRequest_Track: @@ -188,6 +191,8 @@ func (m *Monitor) AcceptRequest(req *rpc.StartEgressRequest) error { cpuHold = m.cpuCostConfig.RoomCompositeCpuCost case *rpc.StartEgressRequest_Web: cpuHold = m.cpuCostConfig.WebCpuCost + case *rpc.StartEgressRequest_Participant: + cpuHold = m.cpuCostConfig.ParticipantCpuCost case *rpc.StartEgressRequest_TrackComposite: cpuHold = m.cpuCostConfig.TrackCompositeCpuCost case *rpc.StartEgressRequest_Track: @@ -206,6 +211,8 @@ func (m *Monitor) EgressStarted(req *rpc.StartEgressRequest) { m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeRoomComposite}).Add(1) case *rpc.StartEgressRequest_Web: m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeWeb}).Add(1) + case *rpc.StartEgressRequest_Participant: + m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeParticipant}).Add(1) case *rpc.StartEgressRequest_TrackComposite: m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeTrackComposite}).Add(1) case *rpc.StartEgressRequest_Track: @@ -224,6 +231,9 @@ func (m *Monitor) EgressEnded(req *rpc.StartEgressRequest) { case *rpc.StartEgressRequest_Web: m.reserved -= m.cpuCostConfig.WebCpuCost m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeWeb}).Sub(1) + case *rpc.StartEgressRequest_Participant: + m.reserved -= m.cpuCostConfig.ParticipantCpuCost + m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeParticipant}).Sub(1) case *rpc.StartEgressRequest_TrackComposite: m.reserved -= m.cpuCostConfig.TrackCompositeCpuCost m.requestGauge.With(prometheus.Labels{"type": types.RequestTypeTrackComposite}).Sub(1) @@ -242,6 +252,8 @@ func (m *Monitor) EgressAborted(req *rpc.StartEgressRequest) { m.reserved -= m.cpuCostConfig.RoomCompositeCpuCost case *rpc.StartEgressRequest_Web: m.reserved -= m.cpuCostConfig.WebCpuCost + case *rpc.StartEgressRequest_Participant: + m.reserved -= m.cpuCostConfig.ParticipantCpuCost case *rpc.StartEgressRequest_TrackComposite: m.reserved -= m.cpuCostConfig.TrackCompositeCpuCost case *rpc.StartEgressRequest_Track: diff --git a/pkg/types/types.go b/pkg/types/types.go index 9424ccff..a0e66d7e 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -26,6 +26,7 @@ const ( // request types RequestTypeRoomComposite = "room_composite" RequestTypeWeb = "web" + RequestTypeParticipant = "participant" RequestTypeTrackComposite = "track_composite" RequestTypeTrack = "track" diff --git a/test/participant.go b/test/participant.go new file mode 100644 index 00000000..6726d4f1 --- /dev/null +++ b/test/participant.go @@ -0,0 +1,274 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package test + +import ( + "testing" + "time" + + "github.com/livekit/egress/pkg/types" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/rpc" + "github.com/livekit/protocol/utils" +) + +func (r *Runner) testParticipant(t *testing.T) { + if !r.runParticipantTests() { + return + } + + r.sourceFramerate = 23.97 + r.testParticipantFile(t) + r.testParticipantStream(t) + r.testParticipantSegments(t) + r.testParticipantMulti(t) +} + +func (r *Runner) runParticipantTest( + t *testing.T, name string, test *testCase, + f func(t *testing.T, identity string), +) { + t.Run(name, func(t *testing.T) { + r.awaitIdle(t) + r.publishSampleOffset(t, test.audioCodec, test.audioDelay, test.audioUnpublish) + if test.audioRepublish != 0 { + r.publishSampleOffset(t, test.audioCodec, test.audioRepublish, 0) + } + r.publishSampleOffset(t, test.videoCodec, test.videoDelay, test.videoUnpublish) + if test.videoRepublish != 0 { + r.publishSampleOffset(t, test.videoCodec, test.videoRepublish, 0) + } + f(t, r.room.LocalParticipant.Identity()) + if t.Failed() { + r.svc.Reset() + go r.svc.Run() + } + }) +} + +func (r *Runner) testParticipantFile(t *testing.T) { + if !r.runFileTests() { + return + } + + t.Run("Participant/File", func(t *testing.T) { + for _, test := range []*testCase{ + { + name: "VP8", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + audioDelay: time.Second * 8, + audioUnpublish: time.Second * 14, + audioRepublish: time.Second * 20, + videoCodec: types.MimeTypeVP8, + filename: "participant_{publisher_identity}_vp8_{time}.mp4", + }, + { + name: "H264", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeH264, + videoDelay: time.Second * 8, + videoUnpublish: time.Second * 14, + videoRepublish: time.Second * 20, + filename: "participant_{room_name}_h264_{time}.mp4", + }, + { + name: "OGG", + fileType: livekit.EncodedFileType_OGG, + audioCodec: types.MimeTypeOpus, + audioUnpublish: time.Second * 10, + audioRepublish: time.Second * 15, + filename: "participant_{room_name}_{time}.ogg", + }, + } { + r.runParticipantTest(t, test.name, test, func(t *testing.T, identity string) { + fileOutput := &livekit.EncodedFileOutput{ + FileType: test.fileType, + Filepath: r.getFilePath(test.filename), + } + if test.filenameSuffix == livekit.SegmentedFileSuffix_INDEX && r.AzureUpload != nil { + fileOutput.Filepath = test.filename + fileOutput.Output = &livekit.EncodedFileOutput_Azure{ + Azure: r.AzureUpload, + } + } + + participantRequest := &livekit.ParticipantEgressRequest{ + RoomName: r.room.Name(), + Identity: identity, + FileOutputs: []*livekit.EncodedFileOutput{fileOutput}, + } + if test.options != nil { + participantRequest.Options = &livekit.ParticipantEgressRequest_Advanced{ + Advanced: test.options, + } + } + + req := &rpc.StartEgressRequest{ + EgressId: utils.NewGuid(utils.EgressPrefix), + Request: &rpc.StartEgressRequest_Participant{ + Participant: participantRequest, + }, + } + + test.expectVideoTranscoding = true + r.runFileTest(t, req, test) + }) + if r.Short { + return + } + } + }) +} + +func (r *Runner) testParticipantStream(t *testing.T) { + if !r.runStreamTests() { + return + } + + test := &testCase{ + audioCodec: types.MimeTypeOpus, + audioDelay: time.Second * 10, + videoCodec: types.MimeTypeVP8, + videoUnpublish: time.Second * 20, + } + + r.runParticipantTest(t, "Participant/Stream", test, + func(t *testing.T, identity string) { + req := &rpc.StartEgressRequest{ + EgressId: utils.NewGuid(utils.EgressPrefix), + Request: &rpc.StartEgressRequest_Participant{ + Participant: &livekit.ParticipantEgressRequest{ + RoomName: r.room.Name(), + Identity: identity, + StreamOutputs: []*livekit.StreamOutput{{ + Urls: []string{streamUrl1, badStreamUrl1}, + }}, + }, + }, + } + + r.runStreamTest(t, req, &testCase{expectVideoTranscoding: true}) + }, + ) +} + +func (r *Runner) testParticipantSegments(t *testing.T) { + if !r.runSegmentTests() { + return + } + + t.Run("Participant/Segments", func(t *testing.T) { + for _, test := range []*testCase{ + { + name: "VP8", + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeVP8, + videoDelay: time.Second * 10, + videoUnpublish: time.Second * 20, + filename: "participant_{publisher_identity}_vp8_{time}", + playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", + }, + { + name: "H264", + audioCodec: types.MimeTypeOpus, + audioDelay: time.Second * 10, + audioUnpublish: time.Second * 20, + videoCodec: types.MimeTypeH264, + filename: "participant_{room_name}_h264_{time}", + playlist: "participant_{room_name}_h264_{time}.m3u8", + }, + } { + r.runParticipantTest(t, test.name, test, + func(t *testing.T, identity string) { + segmentOutput := &livekit.SegmentedFileOutput{ + FilenamePrefix: r.getFilePath(test.filename), + PlaylistName: test.playlist, + FilenameSuffix: test.filenameSuffix, + } + if test.filenameSuffix == livekit.SegmentedFileSuffix_INDEX && r.S3Upload != nil { + segmentOutput.FilenamePrefix = test.filename + segmentOutput.Output = &livekit.SegmentedFileOutput_S3{ + S3: r.S3Upload, + } + } + + trackRequest := &livekit.ParticipantEgressRequest{ + RoomName: r.room.Name(), + Identity: identity, + SegmentOutputs: []*livekit.SegmentedFileOutput{segmentOutput}, + } + if test.options != nil { + trackRequest.Options = &livekit.ParticipantEgressRequest_Advanced{ + Advanced: test.options, + } + } + + req := &rpc.StartEgressRequest{ + EgressId: utils.NewGuid(utils.EgressPrefix), + Request: &rpc.StartEgressRequest_Participant{ + Participant: trackRequest, + }, + } + test.expectVideoTranscoding = true + + r.runSegmentsTest(t, req, test) + }, + ) + if r.Short { + return + } + } + }) +} + +func (r *Runner) testParticipantMulti(t *testing.T) { + if !r.runMultiTests() { + return + } + + test := &testCase{ + audioCodec: types.MimeTypeOpus, + audioUnpublish: time.Second * 20, + videoCodec: types.MimeTypeVP8, + videoDelay: time.Second * 10, + } + + r.runParticipantTest(t, "Participant/Multi", test, + func(t *testing.T, identity string) { + req := &rpc.StartEgressRequest{ + EgressId: utils.NewGuid(utils.EgressPrefix), + Request: &rpc.StartEgressRequest_Participant{ + Participant: &livekit.ParticipantEgressRequest{ + RoomName: r.room.Name(), + Identity: identity, + FileOutputs: []*livekit.EncodedFileOutput{{ + FileType: livekit.EncodedFileType_MP4, + Filepath: r.getFilePath("participant_multiple_{time}"), + }}, + StreamOutputs: []*livekit.StreamOutput{{ + Protocol: livekit.StreamProtocol_RTMP, + }}, + }, + }, + } + + r.runMultipleTest(t, req, false, true, true, livekit.SegmentedFileSuffix_INDEX) + }, + ) +} diff --git a/test/runner.go b/test/runner.go index d0b0cd60..a86eee82 100644 --- a/test/runner.go +++ b/test/runner.go @@ -176,6 +176,7 @@ func (r *Runner) Run(t *testing.T, svc *service.Service, bus psrpc.MessageBus, t // run tests r.testRoomComposite(t) r.testWeb(t) + r.testParticipant(t) r.testTrackComposite(t) r.testTrack(t) } From ba51c395e63ca3d4189f348d68fa5827d2012c9a Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 25 Aug 2023 01:24:51 -0700 Subject: [PATCH 02/20] fixes --- go.mod | 28 ++++++++-------- go.sum | 65 +++++++++++++++++--------------------- pkg/gstreamer/pipeline.go | 2 +- pkg/pipeline/source/sdk.go | 5 +++ 4 files changed, 49 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index 1b4dc365..6346e8d3 100644 --- a/go.mod +++ b/go.mod @@ -19,11 +19,11 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/livekit/livekit-server v1.4.5-0.20230814182001-77c8e824735b github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 - github.com/livekit/protocol v1.6.1-0.20230816204336-79e783089699 - github.com/livekit/psrpc v0.3.2 - github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8 + github.com/livekit/protocol v1.6.2-0.20230825070127-9f0a8f87da8d + github.com/livekit/psrpc v0.3.3 + github.com/livekit/server-sdk-go v1.0.17-0.20230825075524-cb221e2829b0 github.com/pion/rtp v1.8.1 - github.com/pion/webrtc/v3 v3.2.14 + github.com/pion/webrtc/v3 v3.2.16 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.16.0 github.com/stretchr/testify v1.8.4 @@ -39,10 +39,10 @@ require ( ) require ( - cloud.google.com/go v0.110.2 // indirect - cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go v0.110.6 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.0 // indirect + cloud.google.com/go/iam v1.1.1 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bep/debounce v1.2.1 // indirect @@ -69,7 +69,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jxskiss/base62 v1.1.0 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/lithammer/shortuuid/v4 v4.0.0 // indirect github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08 // indirect @@ -79,7 +79,7 @@ require ( github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/nats-io/nats.go v1.26.0 // indirect + github.com/nats-io/nats.go v1.28.0 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/pion/datachannel v1.5.5 // indirect @@ -100,7 +100,7 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect - github.com/redis/go-redis/v9 v9.0.5 // indirect + github.com/redis/go-redis/v9 v9.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/thoas/go-funk v0.9.3 // indirect github.com/twitchtv/twirp v8.1.3+incompatible // indirect @@ -109,7 +109,7 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.10.0 // indirect golang.org/x/crypto v0.12.0 // indirect - golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb // indirect + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.9.0 // indirect golang.org/x/sync v0.3.0 // indirect @@ -118,7 +118,7 @@ require ( golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect + google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect ) diff --git a/go.sum b/go.sum index a1261f0f..622a9c7b 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= -cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= -cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= -cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/storage v1.31.0 h1:+S3LjjEN2zZ+L5hOwj4+1OkGCsLVe0NzpXKQ1pSdTCI= cloud.google.com/go/storage v1.31.0/go.mod h1:81ams1PrhW16L4kF7qg+4mTq7SRs5HsbDTM0bWvrwJ0= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= @@ -36,7 +36,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY= github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0= -github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao= +github.com/bsm/ginkgo/v2 v2.9.5 h1:rtVBYPs3+TC5iLUVOis1B9tjLTup7Cj5IfzosKtvTJ0= github.com/bsm/gomega v1.26.0 h1:LhQm+AFcgV2M0WyKroMASzAzCAJVpAxQXv4SaI9a69Y= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -152,8 +152,8 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -176,12 +176,12 @@ github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 h1:jm09419p0lqTkD github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08 h1:e0qwVjrtzmADgNZpdgSJgyhlF6BgrHkpdnkONL8pLrw= github.com/livekit/mediatransportutil v0.0.0-20230814030822-8d5de0008b08/go.mod h1:xirUXW8xnLGmfCwUeAv/nj1VGo1OO1BmgxrYP7jK/14= -github.com/livekit/protocol v1.6.1-0.20230816204336-79e783089699 h1:8EsXCLF16x00RhpC8rFB/lo9STMj8qWlcknSh+kOVeA= -github.com/livekit/protocol v1.6.1-0.20230816204336-79e783089699/go.mod h1:KBa63PkgSJOAA+qPf22FRzouVYjKIBKfCkFyX04DqVU= -github.com/livekit/psrpc v0.3.2 h1:eAaJhASme33gtoBhCRLH9jsnWcdm1tHWf0WzaDk56ew= -github.com/livekit/psrpc v0.3.2/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= -github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8 h1:Vm64l3U2rWAB2IdG/0/CXIZTgVPmldsQrDAOcq+zQQs= -github.com/livekit/server-sdk-go v1.0.16-0.20230815025737-c12cd2eb8fe8/go.mod h1:4OIkcDpQJuOQJEThr+Z8+WQfkcs12BXC+9cuhigLnDM= +github.com/livekit/protocol v1.6.2-0.20230825070127-9f0a8f87da8d h1:PCFKd1f72grQ38Vgx5IFjUsVcy2SQ0bKxkbVvhkZPWs= +github.com/livekit/protocol v1.6.2-0.20230825070127-9f0a8f87da8d/go.mod h1:/JuO+G/btZ5gNwX2+901L6za3UvVO6DHRXHsv8kkLsU= +github.com/livekit/psrpc v0.3.3 h1:+lltbuN39IdaynXhLLxRShgYqYsRMWeeXKzv60oqyWo= +github.com/livekit/psrpc v0.3.3/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= +github.com/livekit/server-sdk-go v1.0.17-0.20230825075524-cb221e2829b0 h1:Wm/4/ppUgpSJ45rgwnreEnWqeEiKZ9j30I60TF/UgwE= +github.com/livekit/server-sdk-go v1.0.17-0.20230825075524-cb221e2829b0/go.mod h1:lLDZe/p7v4xaVTRMAXXDFWg3PQwGKZQ3hCcLYbVD27k= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= @@ -197,8 +197,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/nats-io/jwt/v2 v2.3.0 h1:z2mA1a7tIf5ShggOFlR1oBPgd6hGqcDYsISxZByUzdI= github.com/nats-io/nats-server/v2 v2.9.8 h1:jgxZsv+A3Reb3MgwxaINcNq/za8xZInKhDg9Q0cGN1o= -github.com/nats-io/nats.go v1.26.0 h1:fWJTYPnZ8DzxIaqIHOAMfColuznchnd5Ab5dbJpgPIE= -github.com/nats-io/nats.go v1.26.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= +github.com/nats-io/nats.go v1.28.0 h1:Th4G6zdsz2d0OqXdfzKLClo6bOfoI/b1kInhRtFIy5c= +github.com/nats-io/nats.go v1.28.0/go.mod h1:XpbWUlOElGwTYbMR7imivs7jJj9GtK7ypv321Wp6pjc= github.com/nats-io/nkeys v0.4.4 h1:xvBJ8d69TznjcQl9t6//Q5xXuVhyYiSos6RPtvQNTwA= github.com/nats-io/nkeys v0.4.4/go.mod h1:XUkxdLPTufzlihbamfzQ7mw/VGx6ObUs+0bN5sNvt64= github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= @@ -218,7 +218,6 @@ github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.3.9/go.mod h1:lT3kv5uUIlHfXHU/ZRD7uKD/ufM202+eTa3C/umgGf4= github.com/pion/ice/v2 v2.3.10 h1:T3bUJKqh7pGEdMyTngUcTeQd6io9X8JjgsVWZDannnY= github.com/pion/ice/v2 v2.3.10/go.mod h1:hHGCibDfmXGqukayQw979xEctASp2Pe5Oe0iDU8pRus= github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= @@ -251,11 +250,10 @@ github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlD github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/turn/v2 v2.1.2/go.mod h1:1kjnPkBcex3dhCU2Am+AAmxDcGhLX3WnMfmkNpvSTQU= github.com/pion/turn/v2 v2.1.3 h1:pYxTVWG2gpC97opdRc5IGsQ1lJ9O/IlNhkzj7MMrGAA= github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= -github.com/pion/webrtc/v3 v3.2.14 h1:GlqnBnnLlcYYA/LOwqLLU1plZYwx0Y/e/57bZ2tzQcU= -github.com/pion/webrtc/v3 v3.2.14/go.mod h1:r1mtixc2MH847mmQTPwlEvGge7D18C2T5qp8jI9Lm44= +github.com/pion/webrtc/v3 v3.2.16 h1:2tfQ8qdyUAjeG5Zn44yE98umMtdxuHembJ3WYhj4Zd4= +github.com/pion/webrtc/v3 v3.2.16/go.mod h1:vm5dipobPQGXn2hNyQ+hh2KbTTTaDxJiDcM+MyAyrsc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -269,8 +267,8 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= -github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o= -github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk= +github.com/redis/go-redis/v9 v9.1.0 h1:137FnGdk+EQdCbye1FW+qOEcY5S+SpY9T0NiuqvtfMY= +github.com/redis/go-redis/v9 v9.1.0/go.mod h1:urWj3He21Dj5k4TK1y59xH8Uj6ATueP8AH1cY3lZl4c= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -322,13 +320,12 @@ golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA= -golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -360,8 +357,6 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= @@ -414,7 +409,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -427,7 +421,6 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -458,12 +451,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= -google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= diff --git a/pkg/gstreamer/pipeline.go b/pkg/gstreamer/pipeline.go index 61e23795..1503ab21 100644 --- a/pkg/gstreamer/pipeline.go +++ b/pkg/gstreamer/pipeline.go @@ -109,7 +109,7 @@ func (p *Pipeline) Run() error { p.OnError(err) return } - logger.Infow("running") + logger.Debugw("starting main loop") p.loop.Run() close(p.running) }) diff --git a/pkg/pipeline/source/sdk.go b/pkg/pipeline/source/sdk.go index 9760add5..52355f16 100644 --- a/pkg/pipeline/source/sdk.go +++ b/pkg/pipeline/source/sdk.go @@ -374,11 +374,15 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo if !s.initialized.IsBroken() { s.mu.Lock() switch s.RequestType { + case types.RequestTypeParticipant: + s.filenameReplacements["{publisher_identity}"] = s.Identity + case types.RequestTypeTrackComposite: if s.Identity == "" || track.Kind() == webrtc.RTPCodecTypeVideo { s.Identity = rp.Identity() s.filenameReplacements["{publisher_identity}"] = s.Identity } + case types.RequestTypeTrack: s.Identity = rp.Identity() s.TrackKind = pub.Kind().String() @@ -489,6 +493,7 @@ func (s *SDKSource) onTrackFinished(trackID string) { active := s.active.Dec() if s.RequestType == types.RequestTypeParticipant { s.callbacks.OnTrackRemoved(trackID) + s.sync.RemoveTrack(trackID) } else if active == 0 { s.onDisconnected() } From 51d18368114a34cb457d4996e7a4cc2c5c8a5ca5 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 25 Aug 2023 13:46:18 -0700 Subject: [PATCH 03/20] fix test svc reset --- test/participant.go | 17 ++++++++++------- test/room_composite.go | 12 +++++++----- test/track_composite.go | 12 +++++++----- test/web.go | 12 +++++++----- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/test/participant.go b/test/participant.go index 6726d4f1..71654ef5 100644 --- a/test/participant.go +++ b/test/participant.go @@ -43,6 +43,13 @@ func (r *Runner) runParticipantTest( f func(t *testing.T, identity string), ) { t.Run(name, func(t *testing.T) { + t.Cleanup(func() { + if t.Failed() { + r.svc.Stop(true) + r.svc.Reset() + go r.svc.Run() + } + }) r.awaitIdle(t) r.publishSampleOffset(t, test.audioCodec, test.audioDelay, test.audioUnpublish) if test.audioRepublish != 0 { @@ -53,10 +60,6 @@ func (r *Runner) runParticipantTest( r.publishSampleOffset(t, test.videoCodec, test.videoRepublish, 0) } f(t, r.room.LocalParticipant.Identity()) - if t.Failed() { - r.svc.Reset() - go r.svc.Run() - } }) } @@ -88,12 +91,12 @@ func (r *Runner) testParticipantFile(t *testing.T) { filename: "participant_{room_name}_h264_{time}.mp4", }, { - name: "OGG", - fileType: livekit.EncodedFileType_OGG, + name: "AudioOnly", + fileType: livekit.EncodedFileType_MP4, audioCodec: types.MimeTypeOpus, audioUnpublish: time.Second * 10, audioRepublish: time.Second * 15, - filename: "participant_{room_name}_{time}.ogg", + filename: "participant_{room_name}_{time}.mp4", }, } { r.runParticipantTest(t, test.name, test, func(t *testing.T, identity string) { diff --git a/test/room_composite.go b/test/room_composite.go index 3e86435a..621d9be1 100644 --- a/test/room_composite.go +++ b/test/room_composite.go @@ -43,14 +43,16 @@ func (r *Runner) testRoomComposite(t *testing.T) { func (r *Runner) runRoomTest(t *testing.T, name string, audioCodec, videoCodec types.MimeType, f func(t *testing.T)) { t.Run(name, func(t *testing.T) { + t.Cleanup(func() { + if t.Failed() { + r.svc.Stop(true) + r.svc.Reset() + go r.svc.Run() + } + }) r.awaitIdle(t) r.publishSamplesToRoom(t, audioCodec, videoCodec) f(t) - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } }) } diff --git a/test/track_composite.go b/test/track_composite.go index 49b0fa46..50241be9 100644 --- a/test/track_composite.go +++ b/test/track_composite.go @@ -42,14 +42,16 @@ func (r *Runner) runTrackTest( f func(t *testing.T, audioTrackID, videoTrackID string), ) { t.Run(name, func(t *testing.T) { + t.Cleanup(func() { + if t.Failed() { + r.svc.Stop(true) + r.svc.Reset() + go r.svc.Run() + } + }) r.awaitIdle(t) audioTrackID, videoTrackID := r.publishSamplesToRoom(t, audioCodec, videoCodec) f(t, audioTrackID, videoTrackID) - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } }) } diff --git a/test/web.go b/test/web.go index 488721a7..e4e08f06 100644 --- a/test/web.go +++ b/test/web.go @@ -38,13 +38,15 @@ func (r *Runner) testWeb(t *testing.T) { func (r *Runner) runWebTest(t *testing.T, name string, f func(t *testing.T)) { t.Run(name, func(t *testing.T) { + t.Cleanup(func() { + if t.Failed() { + r.svc.Stop(true) + r.svc.Reset() + go r.svc.Run() + } + }) r.awaitIdle(t) f(t) - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } }) } From 91ee32406e9bca109f0c676b0f4a1130e86c577b Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 25 Aug 2023 16:11:22 -0700 Subject: [PATCH 04/20] more fixes --- go.mod | 2 +- go.sum | 4 ++-- pkg/pipeline/source/sdk.go | 9 +-------- pkg/pipeline/watch.go | 4 ---- test/integration.go | 2 +- test/participant.go | 7 +++---- test/stream.go | 1 + 7 files changed, 9 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 6346e8d3..630abddb 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/livekit/mageutil v0.0.0-20230125210925-54e8a70427c1 github.com/livekit/protocol v1.6.2-0.20230825070127-9f0a8f87da8d github.com/livekit/psrpc v0.3.3 - github.com/livekit/server-sdk-go v1.0.17-0.20230825075524-cb221e2829b0 + github.com/livekit/server-sdk-go v1.0.17-0.20230825204729-fcf5bdfadd2c github.com/pion/rtp v1.8.1 github.com/pion/webrtc/v3 v3.2.16 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 622a9c7b..257cbe5c 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ github.com/livekit/protocol v1.6.2-0.20230825070127-9f0a8f87da8d h1:PCFKd1f72grQ github.com/livekit/protocol v1.6.2-0.20230825070127-9f0a8f87da8d/go.mod h1:/JuO+G/btZ5gNwX2+901L6za3UvVO6DHRXHsv8kkLsU= github.com/livekit/psrpc v0.3.3 h1:+lltbuN39IdaynXhLLxRShgYqYsRMWeeXKzv60oqyWo= github.com/livekit/psrpc v0.3.3/go.mod h1:n6JntEg+zT6Ji8InoyTpV7wusPNwGqqtxmHlkNhDN0U= -github.com/livekit/server-sdk-go v1.0.17-0.20230825075524-cb221e2829b0 h1:Wm/4/ppUgpSJ45rgwnreEnWqeEiKZ9j30I60TF/UgwE= -github.com/livekit/server-sdk-go v1.0.17-0.20230825075524-cb221e2829b0/go.mod h1:lLDZe/p7v4xaVTRMAXXDFWg3PQwGKZQ3hCcLYbVD27k= +github.com/livekit/server-sdk-go v1.0.17-0.20230825204729-fcf5bdfadd2c h1:tFuVYn9UL6ln6Fwfu0SV3R19tB06OUYluNRxZLtYa+8= +github.com/livekit/server-sdk-go v1.0.17-0.20230825204729-fcf5bdfadd2c/go.mod h1:lLDZe/p7v4xaVTRMAXXDFWg3PQwGKZQ3hCcLYbVD27k= github.com/mackerelio/go-osstat v0.2.4 h1:qxGbdPkFo65PXOb/F/nhDKpF2nGmGaCFDLXoZjJTtUs= github.com/mackerelio/go-osstat v0.2.4/go.mod h1:Zy+qzGdZs3A9cuIqmgbJvwbmLQH9dJvtio5ZjJTbdlQ= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= diff --git a/pkg/pipeline/source/sdk.go b/pkg/pipeline/source/sdk.go index 52355f16..eb08b9e9 100644 --- a/pkg/pipeline/source/sdk.go +++ b/pkg/pipeline/source/sdk.go @@ -198,12 +198,10 @@ func (s *SDKSource) awaitParticipant(identity string) error { return err } - trackCount := 0 - for trackCount == 0 || trackCount < len(rp.Tracks()) { + for trackCount := 0; trackCount == 0 || trackCount < len(rp.Tracks()); trackCount++ { if err = <-s.errors; err != nil { return err } - trackCount++ } s.initialized.Break() @@ -293,10 +291,6 @@ func (s *SDKSource) subscribe(track lksdk.TrackPublication) error { // ----- Callbacks ----- func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.RemoteTrackPublication, rp *lksdk.RemoteParticipant) { - if s.initialized.IsBroken() { - return - } - var onSubscribeErr error defer func() { if s.initialized.IsBroken() { @@ -309,7 +303,6 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo }() s.active.Inc() - ts := &config.TrackSource{ TrackID: pub.SID(), Kind: pub.Kind(), diff --git a/pkg/pipeline/watch.go b/pkg/pipeline/watch.go index ea35be63..4bf61235 100644 --- a/pkg/pipeline/watch.go +++ b/pkg/pipeline/watch.go @@ -221,10 +221,6 @@ func parseDebugInfo(gErr *gst.GError) (element, name, message string) { } func (c *Controller) handleMessageStateChanged(msg *gst.Message) { - if c.playing.IsBroken() { - return - } - _, newState := msg.ParseStateChanged() if newState != gst.StatePlaying { return diff --git a/test/integration.go b/test/integration.go index 557b3373..7c9ee4ab 100644 --- a/test/integration.go +++ b/test/integration.go @@ -259,7 +259,7 @@ func (r *Runner) getUpdate(t *testing.T, egressID string) *livekit.EgressInfo { return info } - case <-time.After(time.Minute): + case <-time.After(time.Second * 30): t.Fatal("no update from results channel") return nil } diff --git a/test/participant.go b/test/participant.go index 71654ef5..f83d0081 100644 --- a/test/participant.go +++ b/test/participant.go @@ -145,10 +145,9 @@ func (r *Runner) testParticipantStream(t *testing.T) { } test := &testCase{ - audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 10, - videoCodec: types.MimeTypeVP8, - videoUnpublish: time.Second * 20, + audioCodec: types.MimeTypeOpus, + audioDelay: time.Second * 10, + videoCodec: types.MimeTypeVP8, } r.runParticipantTest(t, "Participant/Stream", test, diff --git a/test/stream.go b/test/stream.go index 4b9be2b0..3476f8b1 100644 --- a/test/stream.go +++ b/test/stream.go @@ -57,6 +57,7 @@ func (r *Runner) runStreamTest(t *testing.T, req *rpc.StartEgressRequest, test * // verify and check updates time.Sleep(time.Second * 5) r.verifyStreams(t, p, streamUrl1, streamUrl2) + r.checkStreamUpdate(t, egressID, map[string]livekit.StreamInfo_Status{ redactedUrl1: livekit.StreamInfo_ACTIVE, redactedUrl2: livekit.StreamInfo_ACTIVE, From 87b7fbb2d664a58d2e5ef7a20f0741e753642473 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 25 Aug 2023 18:35:27 -0700 Subject: [PATCH 05/20] state updates --- pkg/gstreamer/pipeline.go | 29 ++++++++++++++++++++--------- pkg/pipeline/controller.go | 14 +++++--------- pkg/pipeline/watch.go | 2 +- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/pkg/gstreamer/pipeline.go b/pkg/gstreamer/pipeline.go index 1503ab21..3f924b64 100644 --- a/pkg/gstreamer/pipeline.go +++ b/pkg/gstreamer/pipeline.go @@ -32,6 +32,7 @@ type Pipeline struct { elementsAdded bool started core.Fuse running chan struct{} + stopped core.Fuse } // A pipeline can have either elements or src and sink bins. If you add both you will get a wrong hierarchy error @@ -53,6 +54,7 @@ func NewPipeline(name string, latency uint64, callbacks *Callbacks) (*Pipeline, loop: glib.NewMainLoop(glib.MainContextDefault(), false), started: core.NewFuse(), running: make(chan struct{}), + stopped: core.NewFuse(), }, nil } @@ -97,9 +99,19 @@ func (p *Pipeline) SetWatch(watch func(msg *gst.Message) bool) { } func (p *Pipeline) SetState(state gst.State) error { + p.mu.Lock() + defer p.mu.Unlock() + if err := p.pipeline.SetState(state); err != nil { return errors.ErrGstPipelineError(err) } + if state == gst.StateNull { + for _, src := range p.srcs { + if err := src.SetState(gst.StateNull); err != nil { + return err + } + } + } return nil } @@ -124,15 +136,14 @@ func (p *Pipeline) SendEOS() { } func (p *Pipeline) Stop() { - defer p.loop.Quit() - if err := p.SetState(gst.StateNull); err != nil { - p.OnError(err) - return - } - if err := p.OnStop(); err != nil { - p.OnError(err) - return - } + p.stopped.Once(func() { + defer p.loop.Quit() + + _ = p.SetState(gst.StateNull) + if err := p.OnStop(); err != nil { + p.OnError(err) + } + }) } func (p *Pipeline) DebugBinToDotData(details gst.DebugGraphDetails) string { diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 44cbf20b..f45cec14 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -290,7 +290,7 @@ func (c *Controller) startSessionLimitTimer(ctx context.Context) { if c.playing.IsBroken() { c.SendEOS(ctx) } else { - c.Stop() + c.p.Stop() } }) } @@ -468,13 +468,13 @@ func (c *Controller) SendEOS(ctx context.Context) { case livekit.EgressStatus_EGRESS_ABORTED, livekit.EgressStatus_EGRESS_FAILED: - c.Stop() + c.p.Stop() case livekit.EgressStatus_EGRESS_ACTIVE: c.Info.UpdatedAt = time.Now().UnixNano() if c.Info.Error != "" { c.Info.Status = livekit.EgressStatus_EGRESS_FAILED - c.Stop() + c.p.Stop() } else { c.Info.Status = livekit.EgressStatus_EGRESS_ENDING c.OnUpdate(ctx, c.Info) @@ -495,7 +495,7 @@ func (c *Controller) SendEOS(ctx context.Context) { if c.FinalizationRequired { c.OnError(errors.ErrPipelineFrozen) } else { - c.Stop() + c.p.Stop() } }) @@ -514,11 +514,7 @@ func (c *Controller) OnError(err error) { if c.Info.Error == "" && (!c.eosSent.IsBroken() || c.FinalizationRequired) { c.Info.Error = err.Error() } - go c.Stop() -} - -func (c *Controller) Stop() { - c.stopped.Once(c.p.Stop) + go c.p.Stop() } func (c *Controller) OnStop() error { diff --git a/pkg/pipeline/watch.go b/pkg/pipeline/watch.go index 4bf61235..4ea06011 100644 --- a/pkg/pipeline/watch.go +++ b/pkg/pipeline/watch.go @@ -123,7 +123,7 @@ func (c *Controller) messageWatch(msg *gst.Message) bool { switch msg.Type() { case gst.MessageEOS: logger.Infow("EOS received, stopping pipeline") - c.Stop() + c.p.Stop() return false case gst.MessageWarning: err = c.handleMessageWarning(msg.ParseWarning()) From 80094a263f12f4b486c8648279152c7d2e435dfa Mon Sep 17 00:00:00 2001 From: David Colburn Date: Fri, 25 Aug 2023 21:03:05 -0700 Subject: [PATCH 06/20] ffprobe timeout --- pkg/service/handler.go | 2 +- pkg/service/service.go | 2 +- test/ffprobe.go | 17 +++++++++++++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/service/handler.go b/pkg/service/handler.go index 4c681b1d..a7040d14 100644 --- a/pkg/service/handler.go +++ b/pkg/service/handler.go @@ -208,7 +208,7 @@ func (h *Handler) sendUpdate(ctx context.Context, info *livekit.EgressInfo) { } func sendUpdate(ctx context.Context, c rpc.IOInfoClient, info *livekit.EgressInfo) { - requestType, outputType := egress.GetTypes(info) + requestType, outputType := egress.GetTypes(info.Request) switch info.Status { case livekit.EgressStatus_EGRESS_FAILED: logger.Warnw("egress failed", errors.New(info.Error), diff --git a/pkg/service/service.go b/pkg/service/service.go index af878290..e6fede64 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -128,7 +128,7 @@ func (s *Service) StartEgress(ctx context.Context, req *rpc.StartEgressRequest) return nil, err } - requestType, outputType := egress.GetTypes(p.Info) + requestType, outputType := egress.GetTypes(p.Info.Request) logger.Infow("request validated", "egressID", req.EgressId, "requestType", requestType, diff --git a/test/ffprobe.go b/test/ffprobe.go index a79ea1c9..c20fc780 100644 --- a/test/ffprobe.go +++ b/test/ffprobe.go @@ -105,8 +105,21 @@ func ffprobe(input string) (*FFProbeInfo, error) { } func verify(t *testing.T, in string, p *config.PipelineConfig, res *livekit.EgressInfo, egressType types.EgressType, withMuting bool, sourceFramerate float64) { - info, err := ffprobe(in) - require.NoError(t, err, "input %s does not exist", in) + var info *FFProbeInfo + var err error + + done := make(chan struct{}) + go func() { + info, err = ffprobe(in) + close(done) + }() + + select { + case <-time.After(time.Second * 15): + t.Fatal("no response from ffprobe") + case <-done: + require.NoError(t, err, "input %s does not exist", in) + } switch p.Outputs[egressType].GetOutputType() { case types.OutputTypeRaw: From 55540322d96e889effab479052e891d624eb297b Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 28 Aug 2023 11:38:10 -0700 Subject: [PATCH 07/20] try without delay/unpublish --- test/participant.go | 85 ++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 43 deletions(-) diff --git a/test/participant.go b/test/participant.go index f83d0081..41fe71f3 100644 --- a/test/participant.go +++ b/test/participant.go @@ -18,7 +18,6 @@ package test import ( "testing" - "time" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/livekit" @@ -71,32 +70,32 @@ func (r *Runner) testParticipantFile(t *testing.T) { t.Run("Participant/File", func(t *testing.T) { for _, test := range []*testCase{ { - name: "VP8", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 8, - audioUnpublish: time.Second * 14, - audioRepublish: time.Second * 20, - videoCodec: types.MimeTypeVP8, - filename: "participant_{publisher_identity}_vp8_{time}.mp4", + name: "VP8", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + // audioDelay: time.Second * 8, + // audioUnpublish: time.Second * 14, + // audioRepublish: time.Second * 20, + videoCodec: types.MimeTypeVP8, + filename: "participant_{publisher_identity}_vp8_{time}.mp4", }, { - name: "H264", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeH264, - videoDelay: time.Second * 8, - videoUnpublish: time.Second * 14, - videoRepublish: time.Second * 20, - filename: "participant_{room_name}_h264_{time}.mp4", + name: "H264", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeH264, + // videoDelay: time.Second * 8, + // videoUnpublish: time.Second * 14, + // videoRepublish: time.Second * 20, + filename: "participant_{room_name}_h264_{time}.mp4", }, { - name: "AudioOnly", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - audioUnpublish: time.Second * 10, - audioRepublish: time.Second * 15, - filename: "participant_{room_name}_{time}.mp4", + name: "AudioOnly", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + // audioUnpublish: time.Second * 10, + // audioRepublish: time.Second * 15, + filename: "participant_{room_name}_{time}.mp4", }, } { r.runParticipantTest(t, test.name, test, func(t *testing.T, identity string) { @@ -146,7 +145,7 @@ func (r *Runner) testParticipantStream(t *testing.T) { test := &testCase{ audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 10, + // audioDelay: time.Second * 10, videoCodec: types.MimeTypeVP8, } @@ -178,22 +177,22 @@ func (r *Runner) testParticipantSegments(t *testing.T) { t.Run("Participant/Segments", func(t *testing.T) { for _, test := range []*testCase{ { - name: "VP8", - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeVP8, - videoDelay: time.Second * 10, - videoUnpublish: time.Second * 20, - filename: "participant_{publisher_identity}_vp8_{time}", - playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", + name: "VP8", + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeVP8, + // videoDelay: time.Second * 10, + // videoUnpublish: time.Second * 20, + filename: "participant_{publisher_identity}_vp8_{time}", + playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", }, { - name: "H264", - audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 10, - audioUnpublish: time.Second * 20, - videoCodec: types.MimeTypeH264, - filename: "participant_{room_name}_h264_{time}", - playlist: "participant_{room_name}_h264_{time}.m3u8", + name: "H264", + audioCodec: types.MimeTypeOpus, + // audioDelay: time.Second * 10, + // audioUnpublish: time.Second * 20, + videoCodec: types.MimeTypeH264, + filename: "participant_{room_name}_h264_{time}", + playlist: "participant_{room_name}_h264_{time}.m3u8", }, } { r.runParticipantTest(t, test.name, test, @@ -245,10 +244,10 @@ func (r *Runner) testParticipantMulti(t *testing.T) { } test := &testCase{ - audioCodec: types.MimeTypeOpus, - audioUnpublish: time.Second * 20, - videoCodec: types.MimeTypeVP8, - videoDelay: time.Second * 10, + audioCodec: types.MimeTypeOpus, + // audioUnpublish: time.Second * 20, + videoCodec: types.MimeTypeVP8, + // videoDelay: time.Second * 10, } r.runParticipantTest(t, "Participant/Multi", test, @@ -270,7 +269,7 @@ func (r *Runner) testParticipantMulti(t *testing.T) { }, } - r.runMultipleTest(t, req, false, true, true, livekit.SegmentedFileSuffix_INDEX) + r.runMultipleTest(t, req, true, true, false, livekit.SegmentedFileSuffix_INDEX) }, ) } From 17c34d54696cfc757b2947676f5ab332d74b4869 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 28 Aug 2023 14:16:03 -0700 Subject: [PATCH 08/20] fixes --- magefile.go | 2 ++ pkg/pipeline/builder/video.go | 59 +++++++++++++++++------------------ pkg/service/service.go | 1 + pkg/stats/monitor.go | 7 +++++ test/integration.go | 41 ++++++++++++------------ test/participant.go | 7 ----- test/room_composite.go | 7 ----- test/track_composite.go | 7 ----- test/web.go | 7 ----- 9 files changed, 59 insertions(+), 79 deletions(-) diff --git a/magefile.go b/magefile.go index 5c7b306f..8137e446 100644 --- a/magefile.go +++ b/magefile.go @@ -80,6 +80,8 @@ func Proto() error { } func Integration(configFile string) error { + defer Dotfiles() + dir, err := os.Getwd() if err != nil { return err diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index a9f6670d..14e1e53d 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -225,8 +225,6 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli if err = b.AddElement(h264Parse); err != nil { return err } - - return nil } case types.MimeTypeVP8: @@ -253,8 +251,6 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli if err = b.AddElement(vp8Dec); err != nil { return err } - } else { - return nil } case types.MimeTypeVP9: @@ -300,47 +296,48 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli if err = b.AddElements(vp9Parse, vp9Caps); err != nil { return err } - return nil } default: return errors.ErrNotSupported(string(ts.MimeType)) } - videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) - if err != nil { - return err - } + if p.VideoTranscoding { + videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) + if err != nil { + return err + } - videoConvert, err := gst.NewElement("videoconvert") - if err != nil { - return errors.ErrGstPipelineError(err) - } + videoConvert, err := gst.NewElement("videoconvert") + if err != nil { + return errors.ErrGstPipelineError(err) + } - videoScale, err := gst.NewElement("videoscale") - if err != nil { - return errors.ErrGstPipelineError(err) - } + videoScale, err := gst.NewElement("videoscale") + if err != nil { + return errors.ErrGstPipelineError(err) + } - videoRate, err := gst.NewElement("videorate") - if err != nil { - return errors.ErrGstPipelineError(err) - } - if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { - return err - } + videoRate, err := gst.NewElement("videorate") + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { + return err + } - caps, err := newVideoCapsFilter(p, false) - if err != nil { - return errors.ErrGstPipelineError(err) - } + caps, err := newVideoCapsFilter(p, false) + if err != nil { + return errors.ErrGstPipelineError(err) + } - if err = b.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps); err != nil { - return err + if err = b.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps); err != nil { + return err + } } v.createSrcPad(ts.TrackID) - if err = videoBin.AddSourceBin(b); err != nil { + if err := videoBin.AddSourceBin(b); err != nil { return err } return v.setSelectorPad(ts.TrackID) diff --git a/pkg/service/service.go b/pkg/service/service.go index e6fede64..4391acb3 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -183,6 +183,7 @@ func (s *Service) ListActiveEgress(ctx context.Context, _ *rpc.ListActiveEgressR func (s *Service) Status() ([]byte, error) { info := map[string]interface{}{ "CpuLoad": s.GetCPULoad(), + "CpuHold": s.GetCPUHold(), } s.mu.RLock() diff --git a/pkg/stats/monitor.go b/pkg/stats/monitor.go index d9f72dcf..55aeaa0f 100644 --- a/pkg/stats/monitor.go +++ b/pkg/stats/monitor.go @@ -136,6 +136,13 @@ func (m *Monitor) GetCPULoad() float64 { return (m.cpuStats.NumCPU() - m.cpuStats.GetCPUIdle()) / m.cpuStats.NumCPU() * 100 } +func (m *Monitor) GetCPUHold() float64 { + m.mu.Lock() + defer m.mu.Unlock() + + return m.reserved +} + func (m *Monitor) CanAcceptRequest(req *rpc.StartEgressRequest) bool { m.mu.Lock() defer m.mu.Unlock() diff --git a/test/integration.go b/test/integration.go index 7c9ee4ab..71d008bd 100644 --- a/test/integration.go +++ b/test/integration.go @@ -98,8 +98,7 @@ type testCase struct { func (r *Runner) awaitIdle(t *testing.T) { r.svc.KillAll() for i := 0; i < 30; i++ { - status := r.getStatus(t) - if len(status) == 1 { + if r.svc.GetCPUHold() == 0 { return } time.Sleep(time.Second) @@ -123,25 +122,27 @@ func (r *Runner) publishSamplesToRoom(t *testing.T, audioCodec, videoCodec types } func (r *Runner) publishSampleOffset(t *testing.T, codec types.MimeType, publishAfter, unpublishAfter time.Duration) { - go func() { - time.Sleep(publishAfter) - done := make(chan struct{}) - pub := r.publish(t, codec, done) - if unpublishAfter != 0 { - time.AfterFunc(unpublishAfter, func() { - select { - case <-done: - return - default: + if codec != "" { + go func() { + time.Sleep(publishAfter) + done := make(chan struct{}) + pub := r.publish(t, codec, done) + if unpublishAfter != 0 { + time.AfterFunc(unpublishAfter, func() { + select { + case <-done: + return + default: + _ = r.room.LocalParticipant.UnpublishTrack(pub.SID()) + } + }) + } else { + t.Cleanup(func() { _ = r.room.LocalParticipant.UnpublishTrack(pub.SID()) - } - }) - } else { - t.Cleanup(func() { - _ = r.room.LocalParticipant.UnpublishTrack(pub.SID()) - }) - } - }() + }) + } + }() + } } func (r *Runner) publishSampleToRoom(t *testing.T, codec types.MimeType, withMuting bool) string { diff --git a/test/participant.go b/test/participant.go index 41fe71f3..f25c40e5 100644 --- a/test/participant.go +++ b/test/participant.go @@ -42,13 +42,6 @@ func (r *Runner) runParticipantTest( f func(t *testing.T, identity string), ) { t.Run(name, func(t *testing.T) { - t.Cleanup(func() { - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } - }) r.awaitIdle(t) r.publishSampleOffset(t, test.audioCodec, test.audioDelay, test.audioUnpublish) if test.audioRepublish != 0 { diff --git a/test/room_composite.go b/test/room_composite.go index 621d9be1..00e6ed2c 100644 --- a/test/room_composite.go +++ b/test/room_composite.go @@ -43,13 +43,6 @@ func (r *Runner) testRoomComposite(t *testing.T) { func (r *Runner) runRoomTest(t *testing.T, name string, audioCodec, videoCodec types.MimeType, f func(t *testing.T)) { t.Run(name, func(t *testing.T) { - t.Cleanup(func() { - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } - }) r.awaitIdle(t) r.publishSamplesToRoom(t, audioCodec, videoCodec) f(t) diff --git a/test/track_composite.go b/test/track_composite.go index 50241be9..09e945f9 100644 --- a/test/track_composite.go +++ b/test/track_composite.go @@ -42,13 +42,6 @@ func (r *Runner) runTrackTest( f func(t *testing.T, audioTrackID, videoTrackID string), ) { t.Run(name, func(t *testing.T) { - t.Cleanup(func() { - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } - }) r.awaitIdle(t) audioTrackID, videoTrackID := r.publishSamplesToRoom(t, audioCodec, videoCodec) f(t, audioTrackID, videoTrackID) diff --git a/test/web.go b/test/web.go index e4e08f06..c91b6ede 100644 --- a/test/web.go +++ b/test/web.go @@ -38,13 +38,6 @@ func (r *Runner) testWeb(t *testing.T) { func (r *Runner) runWebTest(t *testing.T, name string, f func(t *testing.T)) { t.Run(name, func(t *testing.T) { - t.Cleanup(func() { - if t.Failed() { - r.svc.Stop(true) - r.svc.Reset() - go r.svc.Run() - } - }) r.awaitIdle(t) f(t) }) From 0985eeb78c2a43a8c50f7fab255bef5f14f2c56d Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 28 Aug 2023 14:56:37 -0700 Subject: [PATCH 09/20] fix unencoded video --- pkg/pipeline/builder/video.go | 147 ++++++++++++++++++++-------------- 1 file changed, 86 insertions(+), 61 deletions(-) diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index 14e1e53d..ba826e8a 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -50,13 +50,13 @@ func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*Vid b := pipeline.NewBin("video") switch p.SourceType { - case types.SourceTypeSDK: - if err := v.buildSDKVideoInput(b, p); err != nil { + case types.SourceTypeWeb: + if err := buildWebVideoInput(b, p); err != nil { return nil, nil, err } - case types.SourceTypeWeb: - if err := buildWebVideoInput(b, p); err != nil { + case types.SourceTypeSDK: + if err := v.buildSDKVideoInput(b, p); err != nil { return nil, nil, err } } @@ -168,15 +168,36 @@ func (v *VideoInput) buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConf } func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) error { + b, err := buildVideoAppSrcBin(videoBin, p, ts) + if err != nil { + return err + } + + if p.VideoTranscoding { + v.createSrcPad(ts.TrackID) + } + + if err = videoBin.AddSourceBin(b); err != nil { + return err + } + + if p.VideoTranscoding { + return v.setSelectorPad(ts.TrackID) + } + + return nil +} + +func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) (*gstreamer.Bin, error) { b := videoBin.NewBin(ts.TrackID) b.SetEOSFunc(ts.EOSFunc) ts.AppSrc.Element.SetArg("format", "time") if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err := b.AddElement(ts.AppSrc.Element); err != nil { - return err + return nil, err } switch ts.MimeType { @@ -185,46 +206,48 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli "application/x-rtp,media=video,payload=%d,encoding-name=H264,clock-rate=%d", ts.PayloadType, ts.ClockRate, ))); err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } rtpH264Depay, err := gst.NewElement("rtph264depay") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } caps, err := gst.NewElement("capsfilter") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = caps.SetProperty("caps", gst.NewCapsFromString( "video/x-h264,stream-format=byte-stream", )); err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElements(rtpH264Depay, caps); err != nil { - return err + return nil, err } if p.VideoTranscoding { avDecH264, err := gst.NewElement("avdec_h264") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElement(avDecH264); err != nil { - return err + return nil, err } } else { h264Parse, err := gst.NewElement("h264parse") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElement(h264Parse); err != nil { - return err + return nil, err } + + return b, nil } case types.MimeTypeVP8: @@ -232,25 +255,27 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli "application/x-rtp,media=video,payload=%d,encoding-name=VP8,clock-rate=%d", ts.PayloadType, ts.ClockRate, ))); err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } rtpVP8Depay, err := gst.NewElement("rtpvp8depay") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElement(rtpVP8Depay); err != nil { - return err + return nil, err } if p.VideoTranscoding { vp8Dec, err := gst.NewElement("vp8dec") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElement(vp8Dec); err != nil { - return err + return nil, err } + } else { + return b, nil } case types.MimeTypeVP9: @@ -258,89 +283,89 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli "application/x-rtp,media=video,payload=%d,encoding-name=VP9,clock-rate=%d", ts.PayloadType, ts.ClockRate, ))); err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } rtpVP9Depay, err := gst.NewElement("rtpvp9depay") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElement(rtpVP9Depay); err != nil { - return err + return nil, err } if p.VideoTranscoding { vp9Dec, err := gst.NewElement("vp9dec") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElement(vp9Dec); err != nil { - return err + return nil, err } } else { vp9Parse, err := gst.NewElement("vp9parse") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } vp9Caps, err := gst.NewElement("capsfilter") if err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = vp9Caps.SetProperty("caps", gst.NewCapsFromString( "video/x-vp9,width=[16,2147483647],height=[16,2147483647]", )); err != nil { - return errors.ErrGstPipelineError(err) + return nil, errors.ErrGstPipelineError(err) } if err = b.AddElements(vp9Parse, vp9Caps); err != nil { - return err + return nil, err } + + return b, nil } default: - return errors.ErrNotSupported(string(ts.MimeType)) + return nil, errors.ErrNotSupported(string(ts.MimeType)) } - if p.VideoTranscoding { - videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) - if err != nil { - return err - } - - videoConvert, err := gst.NewElement("videoconvert") - if err != nil { - return errors.ErrGstPipelineError(err) - } + if err := buildVideoConverter(b, p); err != nil { + return nil, err + } - videoScale, err := gst.NewElement("videoscale") - if err != nil { - return errors.ErrGstPipelineError(err) - } + return b, nil +} - videoRate, err := gst.NewElement("videorate") - if err != nil { - return errors.ErrGstPipelineError(err) - } - if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { - return err - } +func buildVideoConverter(b *gstreamer.Bin, p *config.PipelineConfig) error { + videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) + if err != nil { + return err + } - caps, err := newVideoCapsFilter(p, false) - if err != nil { - return errors.ErrGstPipelineError(err) - } + videoConvert, err := gst.NewElement("videoconvert") + if err != nil { + return errors.ErrGstPipelineError(err) + } - if err = b.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps); err != nil { - return err - } + videoScale, err := gst.NewElement("videoscale") + if err != nil { + return errors.ErrGstPipelineError(err) } - v.createSrcPad(ts.TrackID) - if err := videoBin.AddSourceBin(b); err != nil { + videoRate, err := gst.NewElement("videorate") + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { return err } - return v.setSelectorPad(ts.TrackID) + + caps, err := newVideoCapsFilter(p, false) + if err != nil { + return errors.ErrGstPipelineError(err) + } + + return b.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps) } func (v *VideoInput) buildVideoTestSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig) error { From 61cd92d601974206b1a9ea23fde7eccfedcdd7b2 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Mon, 28 Aug 2023 15:26:11 -0700 Subject: [PATCH 10/20] test delayed publish --- pkg/pipeline/builder/video.go | 38 ++++++++++++++++++----------------- pkg/pipeline/controller.go | 6 +++--- test/participant.go | 13 ++++++------ 3 files changed, 30 insertions(+), 27 deletions(-) diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index ba826e8a..5e83405c 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -34,6 +34,8 @@ import ( const videoTestSrcName = "video_test_src" type VideoInput struct { + bin *gstreamer.Bin + lastPTS atomic.Duration nextPTS atomic.Duration selectedPad string @@ -44,39 +46,39 @@ type VideoInput struct { selector *gst.Element } -func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*VideoInput, *gstreamer.Bin, error) { - v := &VideoInput{} - - b := pipeline.NewBin("video") +func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*VideoInput, error) { + v := &VideoInput{ + bin: pipeline.NewBin("video"), + } switch p.SourceType { case types.SourceTypeWeb: - if err := buildWebVideoInput(b, p); err != nil { - return nil, nil, err + if err := buildWebVideoInput(v.bin, p); err != nil { + return nil, err } case types.SourceTypeSDK: - if err := v.buildSDKVideoInput(b, p); err != nil { - return nil, nil, err + if err := v.buildSDKVideoInput(v.bin, p); err != nil { + return nil, err } } if len(p.Outputs) > 1 { tee, err := gst.NewElementWithName("tee", "video_tee") if err != nil { - return nil, nil, err + return nil, err } - if err = b.AddElement(tee); err != nil { - return nil, nil, err + if err = v.bin.AddElement(tee); err != nil { + return nil, err } } - if err := pipeline.AddSourceBin(b); err != nil { - return nil, nil, err + if err := pipeline.AddSourceBin(v.bin); err != nil { + return nil, err } - return v, b, nil + return v, nil } func buildWebVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { @@ -139,7 +141,7 @@ func (v *VideoInput) buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConf } if p.VideoTrack != nil { - if err := v.AddVideoAppSrcBin(b, p, p.VideoTrack); err != nil { + if err := v.AddVideoAppSrcBin(p, p.VideoTrack); err != nil { return err } } @@ -167,8 +169,8 @@ func (v *VideoInput) buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConf return nil } -func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) error { - b, err := buildVideoAppSrcBin(videoBin, p, ts) +func (v *VideoInput) AddVideoAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { + b, err := buildVideoAppSrcBin(v.bin, p, ts) if err != nil { return err } @@ -177,7 +179,7 @@ func (v *VideoInput) AddVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.Pipeli v.createSrcPad(ts.TrackID) } - if err = videoBin.AddSourceBin(b); err != nil { + if err = v.bin.AddSourceBin(b); err != nil { return err } diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index f45cec14..7e8073b4 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -120,7 +120,7 @@ func (c *Controller) BuildPipeline() error { p.SetWatch(c.messageWatch) p.AddOnStop(c.OnStop) - var audioBin, videoBin *gstreamer.Bin + var audioBin *gstreamer.Bin var videoInput *builder.VideoInput if c.AudioEnabled { @@ -130,7 +130,7 @@ func (c *Controller) BuildPipeline() error { } } if c.VideoEnabled { - videoInput, videoBin, err = builder.BuildVideoBin(p, c.PipelineConfig) + videoInput, err = builder.BuildVideoBin(p, c.PipelineConfig) if err != nil { return err } @@ -143,7 +143,7 @@ func (c *Controller) BuildPipeline() error { p.OnError(err) } case lksdk.TrackKindVideo: - if err := videoInput.AddVideoAppSrcBin(videoBin, c.PipelineConfig, ts); err != nil { + if err := videoInput.AddVideoAppSrcBin(c.PipelineConfig, ts); err != nil { p.OnError(err) } } diff --git a/test/participant.go b/test/participant.go index f25c40e5..96762bbf 100644 --- a/test/participant.go +++ b/test/participant.go @@ -18,6 +18,7 @@ package test import ( "testing" + "time" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/livekit" @@ -66,7 +67,7 @@ func (r *Runner) testParticipantFile(t *testing.T) { name: "VP8", fileType: livekit.EncodedFileType_MP4, audioCodec: types.MimeTypeOpus, - // audioDelay: time.Second * 8, + audioDelay: time.Second * 8, // audioUnpublish: time.Second * 14, // audioRepublish: time.Second * 20, videoCodec: types.MimeTypeVP8, @@ -77,7 +78,7 @@ func (r *Runner) testParticipantFile(t *testing.T) { fileType: livekit.EncodedFileType_MP4, audioCodec: types.MimeTypeOpus, videoCodec: types.MimeTypeH264, - // videoDelay: time.Second * 8, + videoDelay: time.Second * 8, // videoUnpublish: time.Second * 14, // videoRepublish: time.Second * 20, filename: "participant_{room_name}_h264_{time}.mp4", @@ -138,7 +139,7 @@ func (r *Runner) testParticipantStream(t *testing.T) { test := &testCase{ audioCodec: types.MimeTypeOpus, - // audioDelay: time.Second * 10, + audioDelay: time.Second * 10, videoCodec: types.MimeTypeVP8, } @@ -173,7 +174,7 @@ func (r *Runner) testParticipantSegments(t *testing.T) { name: "VP8", audioCodec: types.MimeTypeOpus, videoCodec: types.MimeTypeVP8, - // videoDelay: time.Second * 10, + videoDelay: time.Second * 10, // videoUnpublish: time.Second * 20, filename: "participant_{publisher_identity}_vp8_{time}", playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", @@ -181,7 +182,7 @@ func (r *Runner) testParticipantSegments(t *testing.T) { { name: "H264", audioCodec: types.MimeTypeOpus, - // audioDelay: time.Second * 10, + audioDelay: time.Second * 10, // audioUnpublish: time.Second * 20, videoCodec: types.MimeTypeH264, filename: "participant_{room_name}_h264_{time}", @@ -240,7 +241,7 @@ func (r *Runner) testParticipantMulti(t *testing.T) { audioCodec: types.MimeTypeOpus, // audioUnpublish: time.Second * 20, videoCodec: types.MimeTypeVP8, - // videoDelay: time.Second * 10, + videoDelay: time.Second * 10, } r.runParticipantTest(t, "Participant/Multi", test, From b3f29fdb83b668f0faa5321a8af9adf5c57e9d29 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 29 Aug 2023 11:31:19 -0700 Subject: [PATCH 11/20] test all audio --- pkg/pipeline/builder/audio.go | 121 ++++++++++++++++++---------------- pkg/pipeline/controller.go | 10 +-- test/participant.go | 54 +++++++-------- 3 files changed, 95 insertions(+), 90 deletions(-) diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index 3586d905..89e16486 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -27,17 +27,23 @@ import ( const audioMixerLatency = uint64(2e9) -func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*gstreamer.Bin, error) { - b := pipeline.NewBin("audio") +type AudioBin struct { + bin *gstreamer.Bin +} + +func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*AudioBin, error) { + b := &AudioBin{ + bin: pipeline.NewBin("audio"), + } switch p.SourceType { - case types.SourceTypeSDK: - if err := buildSDKAudioInput(b, p); err != nil { + case types.SourceTypeWeb: + if err := b.buildWebInput(p); err != nil { return nil, err } - case types.SourceTypeWeb: - if err := buildWebAudioInput(b, p); err != nil { + case types.SourceTypeSDK: + if err := b.buildSDKInput(p); err != nil { return nil, err } } @@ -48,19 +54,19 @@ func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*gst return nil, err } - if err = b.AddElement(tee); err != nil { + if err = b.bin.AddElement(tee); err != nil { return nil, err } } - if err := pipeline.AddSourceBin(b); err != nil { + if err := pipeline.AddSourceBin(b.bin); err != nil { return nil, err } return b, nil } -func buildWebAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { pulseSrc, err := gst.NewElement("pulsesrc") if err != nil { return errors.ErrGstPipelineError(err) @@ -68,16 +74,15 @@ func buildWebAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { if err = pulseSrc.SetProperty("device", fmt.Sprintf("%s.monitor", p.Info.EgressId)); err != nil { return errors.ErrGstPipelineError(err) } - if err = b.AddElement(pulseSrc); err != nil { + if err = b.bin.AddElement(pulseSrc); err != nil { return err } - if err = addAudioConverter(b, p); err != nil { + if err = addAudioConverter(b.bin, p); err != nil { return err } - if p.AudioTranscoding { - if err = addAudioEncoder(b, p); err != nil { + if err = b.addEncoder(p); err != nil { return err } } @@ -85,20 +90,20 @@ func buildWebAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { return nil } -func buildSDKAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { if p.AudioTrack != nil { - if err := AddAudioAppSrcBin(b, p, p.AudioTrack); err != nil { + if err := b.AddAudioAppSrcBin(p, p.AudioTrack); err != nil { return err } } - if err := buildAudioTestSrcBin(b, p); err != nil { + if err := b.buildAudioTestSrcBin(p); err != nil { return err } - if err := addAudioMixer(b, p); err != nil { + if err := b.addMixer(p); err != nil { return err } if p.AudioTranscoding { - if err := addAudioEncoder(b, p); err != nil { + if err := b.addEncoder(p); err != nil { return err } } @@ -106,15 +111,15 @@ func buildSDKAudioInput(b *gstreamer.Bin, p *config.PipelineConfig) error { return nil } -func AddAudioAppSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) error { - b := audioBin.NewBin(ts.TrackID) - b.SetEOSFunc(ts.EOSFunc) +func (b *AudioBin) AddAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { + appSrcBin := b.bin.NewBin(ts.TrackID) + appSrcBin.SetEOSFunc(ts.EOSFunc) ts.AppSrc.Element.SetArg("format", "time") if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { return err } - if err := b.AddElement(ts.AppSrc.Element); err != nil { + if err := appSrcBin.AddElement(ts.AppSrc.Element); err != nil { return err } @@ -137,7 +142,7 @@ func AddAudioAppSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig, ts *co return errors.ErrGstPipelineError(err) } - if err = b.AddElements(rtpOpusDepay, opusDec); err != nil { + if err = appSrcBin.AddElements(rtpOpusDepay, opusDec); err != nil { return err } @@ -145,20 +150,20 @@ func AddAudioAppSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig, ts *co return errors.ErrNotSupported(string(ts.MimeType)) } - if err := addAudioConverter(b, p); err != nil { + if err := addAudioConverter(appSrcBin, p); err != nil { return err } - if err := audioBin.AddSourceBin(b); err != nil { + if err := b.bin.AddSourceBin(appSrcBin); err != nil { return err } return nil } -func buildAudioTestSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig) error { - b := audioBin.NewBin("audio_test_src") - if err := audioBin.AddSourceBin(b); err != nil { +func (b *AudioBin) buildAudioTestSrcBin(p *config.PipelineConfig) error { + testSrcBin := b.bin.NewBin("audio_test_src") + if err := b.bin.AddSourceBin(testSrcBin); err != nil { return err } @@ -181,34 +186,10 @@ func buildAudioTestSrcBin(audioBin *gstreamer.Bin, p *config.PipelineConfig) err return err } - return b.AddElements(audioTestSrc, audioCaps) -} - -func addAudioConverter(b *gstreamer.Bin, p *config.PipelineConfig) error { - audioQueue, err := gstreamer.BuildQueue("audio_input_queue", p.Latency, true) - if err != nil { - return err - } - - audioConvert, err := gst.NewElement("audioconvert") - if err != nil { - return errors.ErrGstPipelineError(err) - } - - audioResample, err := gst.NewElement("audioresample") - if err != nil { - return errors.ErrGstPipelineError(err) - } - - capsFilter, err := newAudioCapsFilter(p) - if err != nil { - return err - } - - return b.AddElements(audioQueue, audioConvert, audioResample, capsFilter) + return testSrcBin.AddElements(audioTestSrc, audioCaps) } -func addAudioMixer(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (b *AudioBin) addMixer(p *config.PipelineConfig) error { audioMixer, err := gst.NewElement("audiomixer") if err != nil { return errors.ErrGstPipelineError(err) @@ -222,10 +203,10 @@ func addAudioMixer(b *gstreamer.Bin, p *config.PipelineConfig) error { return err } - return b.AddElements(audioMixer, mixedCaps) + return b.bin.AddElements(audioMixer, mixedCaps) } -func addAudioEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (b *AudioBin) addEncoder(p *config.PipelineConfig) error { switch p.AudioOutCodec { case types.MimeTypeOpus: opusEnc, err := gst.NewElement("opusenc") @@ -235,7 +216,7 @@ func addAudioEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { if err = opusEnc.SetProperty("bitrate", int(p.AudioBitrate*1000)); err != nil { return errors.ErrGstPipelineError(err) } - return b.AddElement(opusEnc) + return b.bin.AddElement(opusEnc) case types.MimeTypeAAC: faac, err := gst.NewElement("faac") @@ -245,7 +226,7 @@ func addAudioEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { if err = faac.SetProperty("bitrate", int(p.AudioBitrate*1000)); err != nil { return errors.ErrGstPipelineError(err) } - return b.AddElement(faac) + return b.bin.AddElement(faac) case types.MimeTypeRawAudio: return nil @@ -255,6 +236,30 @@ func addAudioEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { } } +func addAudioConverter(b *gstreamer.Bin, p *config.PipelineConfig) error { + audioQueue, err := gstreamer.BuildQueue("audio_input_queue", p.Latency, true) + if err != nil { + return err + } + + audioConvert, err := gst.NewElement("audioconvert") + if err != nil { + return errors.ErrGstPipelineError(err) + } + + audioResample, err := gst.NewElement("audioresample") + if err != nil { + return errors.ErrGstPipelineError(err) + } + + capsFilter, err := newAudioCapsFilter(p) + if err != nil { + return err + } + + return b.AddElements(audioQueue, audioConvert, audioResample, capsFilter) +} + func newAudioCapsFilter(p *config.PipelineConfig) (*gst.Element, error) { var caps *gst.Caps switch p.AudioOutCodec { diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 7e8073b4..8b9d4398 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -120,8 +120,8 @@ func (c *Controller) BuildPipeline() error { p.SetWatch(c.messageWatch) p.AddOnStop(c.OnStop) - var audioBin *gstreamer.Bin - var videoInput *builder.VideoInput + var audioBin *builder.AudioBin + var videoBin *builder.VideoInput if c.AudioEnabled { audioBin, err = builder.BuildAudioBin(p, c.PipelineConfig) @@ -130,7 +130,7 @@ func (c *Controller) BuildPipeline() error { } } if c.VideoEnabled { - videoInput, err = builder.BuildVideoBin(p, c.PipelineConfig) + videoBin, err = builder.BuildVideoBin(p, c.PipelineConfig) if err != nil { return err } @@ -139,11 +139,11 @@ func (c *Controller) BuildPipeline() error { p.AddOnTrackAdded(func(ts *config.TrackSource) { switch ts.Kind { case lksdk.TrackKindAudio: - if err := builder.AddAudioAppSrcBin(audioBin, c.PipelineConfig, ts); err != nil { + if err := audioBin.AddAudioAppSrcBin(c.PipelineConfig, ts); err != nil { p.OnError(err) } case lksdk.TrackKindVideo: - if err := videoInput.AddVideoAppSrcBin(c.PipelineConfig, ts); err != nil { + if err := videoBin.AddVideoAppSrcBin(c.PipelineConfig, ts); err != nil { p.OnError(err) } } diff --git a/test/participant.go b/test/participant.go index 96762bbf..efd909ce 100644 --- a/test/participant.go +++ b/test/participant.go @@ -64,32 +64,32 @@ func (r *Runner) testParticipantFile(t *testing.T) { t.Run("Participant/File", func(t *testing.T) { for _, test := range []*testCase{ { - name: "VP8", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 8, - // audioUnpublish: time.Second * 14, - // audioRepublish: time.Second * 20, - videoCodec: types.MimeTypeVP8, - filename: "participant_{publisher_identity}_vp8_{time}.mp4", + name: "VP8", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + audioDelay: time.Second * 8, + audioUnpublish: time.Second * 14, + audioRepublish: time.Second * 20, + videoCodec: types.MimeTypeVP8, + filename: "participant_{publisher_identity}_vp8_{time}.mp4", }, { name: "H264", fileType: livekit.EncodedFileType_MP4, audioCodec: types.MimeTypeOpus, videoCodec: types.MimeTypeH264, - videoDelay: time.Second * 8, + // videoDelay: time.Second * 8, // videoUnpublish: time.Second * 14, // videoRepublish: time.Second * 20, filename: "participant_{room_name}_h264_{time}.mp4", }, { - name: "AudioOnly", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - // audioUnpublish: time.Second * 10, - // audioRepublish: time.Second * 15, - filename: "participant_{room_name}_{time}.mp4", + name: "AudioOnly", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + audioUnpublish: time.Second * 10, + audioRepublish: time.Second * 15, + filename: "participant_{room_name}_{time}.mp4", }, } { r.runParticipantTest(t, test.name, test, func(t *testing.T, identity string) { @@ -174,19 +174,19 @@ func (r *Runner) testParticipantSegments(t *testing.T) { name: "VP8", audioCodec: types.MimeTypeOpus, videoCodec: types.MimeTypeVP8, - videoDelay: time.Second * 10, + // videoDelay: time.Second * 10, // videoUnpublish: time.Second * 20, filename: "participant_{publisher_identity}_vp8_{time}", playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", }, { - name: "H264", - audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 10, - // audioUnpublish: time.Second * 20, - videoCodec: types.MimeTypeH264, - filename: "participant_{room_name}_h264_{time}", - playlist: "participant_{room_name}_h264_{time}.m3u8", + name: "H264", + audioCodec: types.MimeTypeOpus, + audioDelay: time.Second * 10, + audioUnpublish: time.Second * 20, + videoCodec: types.MimeTypeH264, + filename: "participant_{room_name}_h264_{time}", + playlist: "participant_{room_name}_h264_{time}.m3u8", }, } { r.runParticipantTest(t, test.name, test, @@ -238,10 +238,10 @@ func (r *Runner) testParticipantMulti(t *testing.T) { } test := &testCase{ - audioCodec: types.MimeTypeOpus, - // audioUnpublish: time.Second * 20, - videoCodec: types.MimeTypeVP8, - videoDelay: time.Second * 10, + audioCodec: types.MimeTypeOpus, + audioUnpublish: time.Second * 20, + videoCodec: types.MimeTypeVP8, + // videoDelay: time.Second * 10, } r.runParticipantTest(t, "Participant/Multi", test, From 4c60c3f4873766cb804bfd7956812dd9d19af12a Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 29 Aug 2023 14:59:25 -0700 Subject: [PATCH 12/20] more updates --- pkg/config/base.go | 2 +- pkg/gstreamer/bin.go | 100 +++++----- pkg/pipeline/builder/audio.go | 92 ++++----- pkg/pipeline/builder/video.go | 350 ++++++++++++++++++---------------- pkg/pipeline/controller.go | 4 +- test/participant.go | 2 +- 6 files changed, 282 insertions(+), 268 deletions(-) diff --git a/pkg/config/base.go b/pkg/config/base.go index eaa1b64e..d9ff4f91 100644 --- a/pkg/config/base.go +++ b/pkg/config/base.go @@ -96,7 +96,7 @@ func (c *BaseConfig) initLogger(values ...interface{}) error { var gstDebug string switch c.Logging.Level { case "debug": - gstDebug = "3" + gstDebug = "4" case "info", "warn": gstDebug = "2" case "error": diff --git a/pkg/gstreamer/bin.go b/pkg/gstreamer/bin.go index 38d08a9a..1f9abaa2 100644 --- a/pkg/gstreamer/bin.go +++ b/pkg/gstreamer/bin.go @@ -57,68 +57,49 @@ func (b *Bin) NewBin(name string) *Bin { // Add src as a source of b. This should only be called once for each source bin func (b *Bin) AddSourceBin(src *Bin) error { - b.mu.Lock() - defer b.mu.Unlock() - - src.mu.Lock() - alreadyAdded := src.added - src.added = true - src.mu.Unlock() - if alreadyAdded { - return errors.ErrBinAlreadyAdded - } - - b.srcs = append(b.srcs, src) - if err := b.pipeline.Add(src.bin.Element); err != nil { - return errors.ErrGstPipelineError(err) - } - - if b.bin.GetState() == gst.StatePlaying { - if err := src.link(); err != nil { - return err - } - - src.mu.Lock() - err := linkPeersLocked(src, b) - src.mu.Unlock() - if err != nil { - return err - } - - if err = src.bin.SetState(gst.StatePlaying); err != nil { - return err - } - } - - return nil + return b.addBin(src, gst.PadDirectionSource) } // Add src as a sink of b. This should only be called once for each sink bin func (b *Bin) AddSinkBin(sink *Bin) error { + return b.addBin(sink, gst.PadDirectionSink) +} + +func (b *Bin) addBin(bin *Bin, direction gst.PadDirection) error { b.mu.Lock() defer b.mu.Unlock() - sink.mu.Lock() - alreadyAdded := sink.added - sink.added = true - sink.mu.Unlock() + bin.mu.Lock() + alreadyAdded := bin.added + bin.added = true + bin.mu.Unlock() if alreadyAdded { return errors.ErrBinAlreadyAdded } - b.sinks = append(b.sinks, sink) - if err := b.pipeline.Add(sink.bin.Element); err != nil { + if direction == gst.PadDirectionSource { + b.srcs = append(b.srcs, bin) + } else { + b.sinks = append(b.sinks, bin) + } + + if err := b.pipeline.Add(bin.bin.Element); err != nil { return errors.ErrGstPipelineError(err) } if b.bin.GetState() == gst.StatePlaying { - if err := sink.link(); err != nil { + if err := bin.link(); err != nil { return err } - sink.mu.Lock() - err := linkPeersLocked(b, sink) - sink.mu.Unlock() + var err error + bin.mu.Lock() + if direction == gst.PadDirectionSource { + err = linkPeersLocked(bin, b) + } else { + err = linkPeersLocked(b, bin) + } + bin.mu.Unlock() if err != nil { return err } @@ -369,15 +350,30 @@ func linkPeersLocked(src, sink *Bin) error { return err } - if src.bin.GetState() == gst.StatePlaying { - srcPad.AddProbe(gst.PadProbeTypeBlockDownstream, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { - if err = sink.bin.SetState(gst.StatePlaying); err != nil { - src.OnError(errors.ErrGstPipelineError(err)) - return gst.PadProbeUnhandled - } + srcState := src.bin.GetState() + sinkState := sink.bin.GetState() - return gst.PadProbeRemove - }) + if srcState != sinkState { + if sinkState == gst.StatePlaying { + srcPad.AddProbe(gst.PadProbeTypeBlockDownstream, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { + if padReturn := srcPad.Link(sinkPad.Pad); padReturn != gst.PadLinkOK { + logger.Errorw("failed to link", errors.ErrPadLinkFailed(src.bin.GetName(), sink.bin.GetName(), padReturn.String())) + } + return gst.PadProbeRemove + }) + return src.bin.SetState(gst.StatePlaying) + } + + if srcState == gst.StatePlaying { + srcPad.AddProbe(gst.PadProbeTypeBlockDownstream, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { + if err = sink.bin.SetState(gst.StatePlaying); err != nil { + src.OnError(errors.ErrGstPipelineError(err)) + return gst.PadProbeUnhandled + } + + return gst.PadProbeRemove + }) + } } if padReturn := srcPad.Link(sinkPad.Pad); padReturn != gst.PadLinkOK { diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index 89e16486..db498856 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -66,51 +66,6 @@ func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*Aud return b, nil } -func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { - pulseSrc, err := gst.NewElement("pulsesrc") - if err != nil { - return errors.ErrGstPipelineError(err) - } - if err = pulseSrc.SetProperty("device", fmt.Sprintf("%s.monitor", p.Info.EgressId)); err != nil { - return errors.ErrGstPipelineError(err) - } - if err = b.bin.AddElement(pulseSrc); err != nil { - return err - } - - if err = addAudioConverter(b.bin, p); err != nil { - return err - } - if p.AudioTranscoding { - if err = b.addEncoder(p); err != nil { - return err - } - } - - return nil -} - -func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { - if p.AudioTrack != nil { - if err := b.AddAudioAppSrcBin(p, p.AudioTrack); err != nil { - return err - } - } - if err := b.buildAudioTestSrcBin(p); err != nil { - return err - } - if err := b.addMixer(p); err != nil { - return err - } - if p.AudioTranscoding { - if err := b.addEncoder(p); err != nil { - return err - } - } - - return nil -} - func (b *AudioBin) AddAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { appSrcBin := b.bin.NewBin(ts.TrackID) appSrcBin.SetEOSFunc(ts.EOSFunc) @@ -161,7 +116,52 @@ func (b *AudioBin) AddAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackS return nil } -func (b *AudioBin) buildAudioTestSrcBin(p *config.PipelineConfig) error { +func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { + pulseSrc, err := gst.NewElement("pulsesrc") + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = pulseSrc.SetProperty("device", fmt.Sprintf("%s.monitor", p.Info.EgressId)); err != nil { + return errors.ErrGstPipelineError(err) + } + if err = b.bin.AddElement(pulseSrc); err != nil { + return err + } + + if err = addAudioConverter(b.bin, p); err != nil { + return err + } + if p.AudioTranscoding { + if err = b.addEncoder(p); err != nil { + return err + } + } + + return nil +} + +func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { + if p.AudioTrack != nil { + if err := b.AddAudioAppSrcBin(p, p.AudioTrack); err != nil { + return err + } + } + if err := b.addAudioTestSrcBin(p); err != nil { + return err + } + if err := b.addMixer(p); err != nil { + return err + } + if p.AudioTranscoding { + if err := b.addEncoder(p); err != nil { + return err + } + } + + return nil +} + +func (b *AudioBin) addAudioTestSrcBin(p *config.PipelineConfig) error { testSrcBin := b.bin.NewBin("audio_test_src") if err := b.bin.AddSourceBin(testSrcBin); err != nil { return err diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index 5e83405c..55b4d7ea 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -33,7 +33,7 @@ import ( const videoTestSrcName = "video_test_src" -type VideoInput struct { +type VideoBin struct { bin *gstreamer.Bin lastPTS atomic.Duration @@ -46,19 +46,19 @@ type VideoInput struct { selector *gst.Element } -func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*VideoInput, error) { - v := &VideoInput{ +func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*VideoBin, error) { + b := &VideoBin{ bin: pipeline.NewBin("video"), } switch p.SourceType { case types.SourceTypeWeb: - if err := buildWebVideoInput(v.bin, p); err != nil { + if err := b.buildWebInput(p); err != nil { return nil, err } case types.SourceTypeSDK: - if err := v.buildSDKVideoInput(v.bin, p); err != nil { + if err := b.buildSDKInput(p); err != nil { return nil, err } } @@ -69,19 +69,138 @@ func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*Vid return nil, err } - if err = v.bin.AddElement(tee); err != nil { + if err = b.bin.AddElement(tee); err != nil { return nil, err } } - if err := pipeline.AddSourceBin(v.bin); err != nil { + if err := pipeline.AddSourceBin(b.bin); err != nil { return nil, err } - return v, nil + return b, nil +} + +func (b *VideoBin) AddVideoAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { + appSrcBin, err := buildVideoAppSrcBin(b.bin, p, ts) + if err != nil { + return err + } + + if p.VideoTranscoding { + b.createSrcPad(ts.TrackID) + } + + if err = b.bin.AddSourceBin(appSrcBin); err != nil { + return err + } + + if p.VideoTranscoding { + return b.setSelectorPad(ts.TrackID) + } + + return nil +} + +func (b *VideoBin) getSrcPad(name string) *gst.Pad { + b.mu.Lock() + defer b.mu.Unlock() + + return b.pads[name] +} + +func (b *VideoBin) onTrackMuted(trackID string) { + if b.selectedPad != trackID { + return + } + + if err := b.setSelectorPad(videoTestSrcName); err != nil { + logger.Errorw("failed to set selector pad", err) + } +} + +func (b *VideoBin) onTrackUnmuted(trackID string, pts time.Duration) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.pads[trackID] == nil { + return + } + + b.nextPTS.Store(pts) + b.nextPad = trackID +} + +func (b *VideoBin) createSrcPad(trackID string) { + b.mu.Lock() + defer b.mu.Unlock() + + pad := b.selector.GetRequestPad("sink_%u") + pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + buffer := info.GetBuffer() + for b.nextPTS.Load() != 0 { + time.Sleep(time.Millisecond * 100) + } + if buffer.PresentationTimestamp() < b.lastPTS.Load() { + return gst.PadProbeDrop + } + b.lastPTS.Store(buffer.PresentationTimestamp()) + return gst.PadProbeOK + }) + b.pads[trackID] = pad } -func buildWebVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (b *VideoBin) createTestSrcPad() { + b.mu.Lock() + defer b.mu.Unlock() + + pad := b.selector.GetRequestPad("sink_%u") + pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + buffer := info.GetBuffer() + if buffer.PresentationTimestamp() < b.lastPTS.Load() { + return gst.PadProbeDrop + } + if nextPTS := b.nextPTS.Load(); nextPTS != 0 && buffer.PresentationTimestamp() >= nextPTS { + if err := b.setSelectorPad(b.nextPad); err != nil { + logger.Errorw("failed to unmute", err) + return gst.PadProbeDrop + } + b.nextPad = "" + b.nextPTS.Store(0) + } + b.lastPTS.Store(buffer.PresentationTimestamp()) + return gst.PadProbeOK + }) + b.pads[videoTestSrcName] = pad +} + +// TODO: go-gst should accept objects directly and handle conversion to C +func (b *VideoBin) setSelectorPad(name string) error { + b.mu.Lock() + defer b.mu.Unlock() + + pad := b.pads[name] + + pt, err := b.selector.GetPropertyType("active-pad") + if err != nil { + return errors.ErrGstPipelineError(err) + } + + val, err := glib.ValueInit(pt) + if err != nil { + return errors.ErrGstPipelineError(err) + } + val.SetInstance(uintptr(unsafe.Pointer(pad.Instance()))) + + if err = b.selector.SetPropertyValue("active-pad", val); err != nil { + return errors.ErrGstPipelineError(err) + } + + b.selectedPad = name + return nil +} + +func (b *VideoBin) buildWebInput(p *config.PipelineConfig) error { xImageSrc, err := gst.NewElement("ximagesrc") if err != nil { return errors.ErrGstPipelineError(err) @@ -118,50 +237,50 @@ func buildWebVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { return errors.ErrGstPipelineError(err) } - if err = b.AddElements(xImageSrc, videoQueue, videoConvert, caps); err != nil { + if err = b.bin.AddElements(xImageSrc, videoQueue, videoConvert, caps); err != nil { return err } if p.VideoTranscoding { - if err = addVideoEncoder(b, p); err != nil { + if err = b.addEncoder(p); err != nil { return err } } return nil } -func (v *VideoInput) buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConfig) error { - v.pads = make(map[string]*gst.Pad) +func (b *VideoBin) buildSDKInput(p *config.PipelineConfig) error { + b.pads = make(map[string]*gst.Pad) // add selector first so pads can be created if p.VideoTranscoding { - if err := v.addVideoSelector(b, p); err != nil { + if err := b.addSelector(p); err != nil { return err } } if p.VideoTrack != nil { - if err := v.AddVideoAppSrcBin(p, p.VideoTrack); err != nil { + if err := b.AddVideoAppSrcBin(p, p.VideoTrack); err != nil { return err } } if p.VideoTranscoding { - if err := v.buildVideoTestSrcBin(b, p); err != nil { + if err := b.addVideoTestSrcBin(p); err != nil { return err } if p.VideoTrack == nil { - if err := v.setSelectorPad(videoTestSrcName); err != nil { + if err := b.setSelectorPad(videoTestSrcName); err != nil { return err } } - b.SetGetSrcPad(v.getSrcPad) - b.Callbacks.AddOnTrackMuted(v.onTrackMuted) - b.Callbacks.AddOnTrackUnmuted(v.onTrackUnmuted) + b.bin.SetGetSrcPad(b.getSrcPad) + b.bin.Callbacks.AddOnTrackMuted(b.onTrackMuted) + b.bin.Callbacks.AddOnTrackUnmuted(b.onTrackUnmuted) - if err := addVideoEncoder(b, p); err != nil { + if err := b.addEncoder(p); err != nil { return err } } @@ -169,27 +288,6 @@ func (v *VideoInput) buildSDKVideoInput(b *gstreamer.Bin, p *config.PipelineConf return nil } -func (v *VideoInput) AddVideoAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { - b, err := buildVideoAppSrcBin(v.bin, p, ts) - if err != nil { - return err - } - - if p.VideoTranscoding { - v.createSrcPad(ts.TrackID) - } - - if err = v.bin.AddSourceBin(b); err != nil { - return err - } - - if p.VideoTranscoding { - return v.setSelectorPad(ts.TrackID) - } - - return nil -} - func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) (*gstreamer.Bin, error) { b := videoBin.NewBin(ts.TrackID) b.SetEOSFunc(ts.EOSFunc) @@ -331,93 +429,76 @@ func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts * return nil, errors.ErrNotSupported(string(ts.MimeType)) } - if err := buildVideoConverter(b, p); err != nil { + if err := addVideoConverter(b, p); err != nil { return nil, err } return b, nil } -func buildVideoConverter(b *gstreamer.Bin, p *config.PipelineConfig) error { - videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) - if err != nil { +func (b *VideoBin) addVideoTestSrcBin(p *config.PipelineConfig) error { + testSrcBin := b.bin.NewBin(videoTestSrcName) + if err := b.bin.AddSourceBin(testSrcBin); err != nil { return err } - videoConvert, err := gst.NewElement("videoconvert") + videoTestSrc, err := gst.NewElement("videotestsrc") if err != nil { return errors.ErrGstPipelineError(err) } - - videoScale, err := gst.NewElement("videoscale") - if err != nil { + if err = videoTestSrc.SetProperty("is-live", true); err != nil { return errors.ErrGstPipelineError(err) } + videoTestSrc.SetArg("pattern", "black") - videoRate, err := gst.NewElement("videorate") + caps, err := newVideoCapsFilter(p, true) if err != nil { return errors.ErrGstPipelineError(err) } - if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { - return err - } - caps, err := newVideoCapsFilter(p, false) - if err != nil { - return errors.ErrGstPipelineError(err) + if err = testSrcBin.AddElements(videoTestSrc, caps); err != nil { + return err } - return b.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps) + b.createTestSrcPad() + return nil } -func (v *VideoInput) buildVideoTestSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig) error { - b := videoBin.NewBin(videoTestSrcName) - if err := videoBin.AddSourceBin(b); err != nil { - return err - } - - videoTestSrc, err := gst.NewElement("videotestsrc") +func (b *VideoBin) addSelector(p *config.PipelineConfig) error { + inputSelector, err := gst.NewElement("input-selector") if err != nil { return errors.ErrGstPipelineError(err) } - if err = videoTestSrc.SetProperty("is-live", true); err != nil { - return errors.ErrGstPipelineError(err) - } - videoTestSrc.SetArg("pattern", "black") - - caps, err := newVideoCapsFilter(p, true) + videoRate, err := gst.NewElement("videorate") if err != nil { return errors.ErrGstPipelineError(err) } - - if err = b.AddElements(videoTestSrc, caps); err != nil { + if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { + return err + } + if err = videoRate.SetProperty("skip-to-first", true); err != nil { return err } - v.createTestSrcPad() - return nil -} - -func (v *VideoInput) addVideoSelector(b *gstreamer.Bin, p *config.PipelineConfig) error { - inputSelector, err := gst.NewElement("input-selector") + caps, err := newVideoCapsFilter(p, true) if err != nil { return errors.ErrGstPipelineError(err) } - if err = b.AddElements(inputSelector); err != nil { + if err = b.bin.AddElements(inputSelector, videoRate, caps); err != nil { return err } - v.selector = inputSelector + b.selector = inputSelector return nil } -func addVideoEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { +func (b *VideoBin) addEncoder(p *config.PipelineConfig) error { videoQueue, err := gstreamer.BuildQueue("video_encoder_queue", p.Latency, false) if err != nil { return err } - if err = b.AddElement(videoQueue); err != nil { + if err = b.bin.AddElement(videoQueue); err != nil { return err } @@ -460,7 +541,7 @@ func addVideoEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { return errors.ErrGstPipelineError(err) } - if err = b.AddElements(x264Enc, caps); err != nil { + if err = b.bin.AddElements(x264Enc, caps); err != nil { return err } @@ -492,7 +573,7 @@ func addVideoEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { if err = vp9Enc.SetProperty("min-quantizer", 2); err != nil { return errors.ErrGstPipelineError(err) } - if err = b.AddElement(vp9Enc); err != nil { + if err = b.bin.AddElement(vp9Enc); err != nil { return err } @@ -503,102 +584,39 @@ func addVideoEncoder(b *gstreamer.Bin, p *config.PipelineConfig) error { } } -func (v *VideoInput) getSrcPad(name string) *gst.Pad { - v.mu.Lock() - defer v.mu.Unlock() - - return v.pads[name] -} - -func (v *VideoInput) createSrcPad(trackID string) { - v.mu.Lock() - defer v.mu.Unlock() - - pad := v.selector.GetRequestPad("sink_%u") - pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - buffer := info.GetBuffer() - for v.nextPTS.Load() != 0 { - time.Sleep(time.Millisecond * 100) - } - if buffer.PresentationTimestamp() < v.lastPTS.Load() { - return gst.PadProbeDrop - } - v.lastPTS.Store(buffer.PresentationTimestamp()) - return gst.PadProbeOK - }) - v.pads[trackID] = pad -} - -func (v *VideoInput) createTestSrcPad() { - v.mu.Lock() - defer v.mu.Unlock() - - pad := v.selector.GetRequestPad("sink_%u") - pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - buffer := info.GetBuffer() - if buffer.PresentationTimestamp() < v.lastPTS.Load() { - return gst.PadProbeDrop - } - if nextPTS := v.nextPTS.Load(); nextPTS != 0 && buffer.PresentationTimestamp() >= nextPTS { - if err := v.setSelectorPad(v.nextPad); err != nil { - logger.Errorw("failed to unmute", err) - return gst.PadProbeDrop - } - v.nextPad = "" - v.nextPTS.Store(0) - } - v.lastPTS.Store(buffer.PresentationTimestamp()) - return gst.PadProbeOK - }) - v.pads[videoTestSrcName] = pad -} - -func (v *VideoInput) onTrackMuted(trackID string) { - if v.selectedPad != trackID { - return - } - - if err := v.setSelectorPad(videoTestSrcName); err != nil { - logger.Errorw("failed to set selector pad", err) +func addVideoConverter(b *gstreamer.Bin, p *config.PipelineConfig) error { + videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) + if err != nil { + return err } -} -func (v *VideoInput) onTrackUnmuted(trackID string, pts time.Duration) { - v.mu.Lock() - defer v.mu.Unlock() - - if v.pads[trackID] == nil { - return + videoConvert, err := gst.NewElement("videoconvert") + if err != nil { + return errors.ErrGstPipelineError(err) } - v.nextPTS.Store(pts) - v.nextPad = trackID -} - -// TODO: go-gst should accept objects directly and handle conversion to C -func (v *VideoInput) setSelectorPad(name string) error { - v.mu.Lock() - defer v.mu.Unlock() - - pad := v.pads[name] - - pt, err := v.selector.GetPropertyType("active-pad") + videoScale, err := gst.NewElement("videoscale") if err != nil { return errors.ErrGstPipelineError(err) } - val, err := glib.ValueInit(pt) + videoRate, err := gst.NewElement("videorate") if err != nil { return errors.ErrGstPipelineError(err) } - val.SetInstance(uintptr(unsafe.Pointer(pad.Instance()))) + if err = videoRate.SetProperty("max-duplication-time", uint64(time.Second)); err != nil { + return err + } + if err = videoRate.SetProperty("skip-to-first", true); err != nil { + return err + } - if err = v.selector.SetPropertyValue("active-pad", val); err != nil { + caps, err := newVideoCapsFilter(p, true) + if err != nil { return errors.ErrGstPipelineError(err) } - v.selectedPad = name - return nil + return b.AddElements(videoQueue, videoConvert, videoScale, videoRate, caps) } func newVideoCapsFilter(p *config.PipelineConfig, includeFramerate bool) (*gst.Element, error) { diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 8b9d4398..484cef91 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -84,7 +84,7 @@ func New(ctx context.Context, conf *config.PipelineConfig) (*Controller, error) _, span := tracer.Start(ctx, "gst.Init") defer span.End() gst.Init(nil) - gst.SetLogFunction(c.gstLog) + // gst.SetLogFunction(c.gstLog) close(c.callbacks.GstReady) }() @@ -121,7 +121,7 @@ func (c *Controller) BuildPipeline() error { p.AddOnStop(c.OnStop) var audioBin *builder.AudioBin - var videoBin *builder.VideoInput + var videoBin *builder.VideoBin if c.AudioEnabled { audioBin, err = builder.BuildAudioBin(p, c.PipelineConfig) diff --git a/test/participant.go b/test/participant.go index efd909ce..32fbbdad 100644 --- a/test/participant.go +++ b/test/participant.go @@ -78,7 +78,7 @@ func (r *Runner) testParticipantFile(t *testing.T) { fileType: livekit.EncodedFileType_MP4, audioCodec: types.MimeTypeOpus, videoCodec: types.MimeTypeH264, - // videoDelay: time.Second * 8, + videoDelay: time.Second * 8, // videoUnpublish: time.Second * 14, // videoRepublish: time.Second * 20, filename: "participant_{room_name}_h264_{time}.mp4", From 8673318745304891081551f4a38fa4a1460d2f65 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 29 Aug 2023 15:25:47 -0700 Subject: [PATCH 13/20] final? --- pkg/config/base.go | 2 +- pkg/pipeline/controller.go | 2 +- test/participant.go | 32 ++++++++++++++++---------------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/pkg/config/base.go b/pkg/config/base.go index d9ff4f91..eaa1b64e 100644 --- a/pkg/config/base.go +++ b/pkg/config/base.go @@ -96,7 +96,7 @@ func (c *BaseConfig) initLogger(values ...interface{}) error { var gstDebug string switch c.Logging.Level { case "debug": - gstDebug = "4" + gstDebug = "3" case "info", "warn": gstDebug = "2" case "error": diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 484cef91..ef444cc1 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -84,7 +84,7 @@ func New(ctx context.Context, conf *config.PipelineConfig) (*Controller, error) _, span := tracer.Start(ctx, "gst.Init") defer span.End() gst.Init(nil) - // gst.SetLogFunction(c.gstLog) + gst.SetLogFunction(c.gstLog) close(c.callbacks.GstReady) }() diff --git a/test/participant.go b/test/participant.go index 32fbbdad..0201e58e 100644 --- a/test/participant.go +++ b/test/participant.go @@ -74,14 +74,14 @@ func (r *Runner) testParticipantFile(t *testing.T) { filename: "participant_{publisher_identity}_vp8_{time}.mp4", }, { - name: "H264", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeH264, - videoDelay: time.Second * 8, - // videoUnpublish: time.Second * 14, - // videoRepublish: time.Second * 20, - filename: "participant_{room_name}_h264_{time}.mp4", + name: "H264", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeH264, + videoDelay: time.Second * 8, + videoUnpublish: time.Second * 14, + videoRepublish: time.Second * 20, + filename: "participant_{room_name}_h264_{time}.mp4", }, { name: "AudioOnly", @@ -171,13 +171,13 @@ func (r *Runner) testParticipantSegments(t *testing.T) { t.Run("Participant/Segments", func(t *testing.T) { for _, test := range []*testCase{ { - name: "VP8", - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeVP8, - // videoDelay: time.Second * 10, - // videoUnpublish: time.Second * 20, - filename: "participant_{publisher_identity}_vp8_{time}", - playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", + name: "VP8", + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeVP8, + videoDelay: time.Second * 10, + videoUnpublish: time.Second * 20, + filename: "participant_{publisher_identity}_vp8_{time}", + playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", }, { name: "H264", @@ -241,7 +241,7 @@ func (r *Runner) testParticipantMulti(t *testing.T) { audioCodec: types.MimeTypeOpus, audioUnpublish: time.Second * 20, videoCodec: types.MimeTypeVP8, - // videoDelay: time.Second * 10, + videoDelay: time.Second * 10, } r.runParticipantTest(t, "Participant/Multi", test, From 8e5c5a64253896b51be052e72a27686736a516a1 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 29 Aug 2023 16:39:05 -0700 Subject: [PATCH 14/20] segments with video delay broken --- test/participant.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/participant.go b/test/participant.go index 0201e58e..bbf96244 100644 --- a/test/participant.go +++ b/test/participant.go @@ -171,13 +171,13 @@ func (r *Runner) testParticipantSegments(t *testing.T) { t.Run("Participant/Segments", func(t *testing.T) { for _, test := range []*testCase{ { - name: "VP8", - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeVP8, - videoDelay: time.Second * 10, - videoUnpublish: time.Second * 20, - filename: "participant_{publisher_identity}_vp8_{time}", - playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", + name: "VP8", + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeVP8, + // videoDelay: time.Second * 10, + // videoUnpublish: time.Second * 20, + filename: "participant_{publisher_identity}_vp8_{time}", + playlist: "participant_{publisher_identity}_vp8_{time}.m3u8", }, { name: "H264", From 40d9d71125c7e203eeb082690674a3af9eb6a8a4 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Tue, 29 Aug 2023 19:09:19 -0700 Subject: [PATCH 15/20] test with video delays --- test/participant.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/participant.go b/test/participant.go index bbf96244..a2e88cba 100644 --- a/test/participant.go +++ b/test/participant.go @@ -74,14 +74,14 @@ func (r *Runner) testParticipantFile(t *testing.T) { filename: "participant_{publisher_identity}_vp8_{time}.mp4", }, { - name: "H264", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeH264, - videoDelay: time.Second * 8, - videoUnpublish: time.Second * 14, - videoRepublish: time.Second * 20, - filename: "participant_{room_name}_h264_{time}.mp4", + name: "H264", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeH264, + videoDelay: time.Second * 8, + // videoUnpublish: time.Second * 14, + // videoRepublish: time.Second * 20, + filename: "participant_{room_name}_h264_{time}.mp4", }, { name: "AudioOnly", From d756b9cd72a15dbf09974bd8149a7c3165555abd Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 30 Aug 2023 13:11:15 -0700 Subject: [PATCH 16/20] passing locally --- test/participant.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/participant.go b/test/participant.go index a2e88cba..d57bfe27 100644 --- a/test/participant.go +++ b/test/participant.go @@ -74,14 +74,14 @@ func (r *Runner) testParticipantFile(t *testing.T) { filename: "participant_{publisher_identity}_vp8_{time}.mp4", }, { - name: "H264", - fileType: livekit.EncodedFileType_MP4, - audioCodec: types.MimeTypeOpus, - videoCodec: types.MimeTypeH264, - videoDelay: time.Second * 8, - // videoUnpublish: time.Second * 14, - // videoRepublish: time.Second * 20, - filename: "participant_{room_name}_h264_{time}.mp4", + name: "H264", + fileType: livekit.EncodedFileType_MP4, + audioCodec: types.MimeTypeOpus, + videoCodec: types.MimeTypeH264, + videoDelay: time.Second * 8, + videoUnpublish: time.Second * 14, + videoRepublish: time.Second * 20, + filename: "participant_{room_name}_h264_{time}.mp4", }, { name: "AudioOnly", @@ -139,7 +139,7 @@ func (r *Runner) testParticipantStream(t *testing.T) { test := &testCase{ audioCodec: types.MimeTypeOpus, - audioDelay: time.Second * 10, + audioDelay: time.Second * 8, videoCodec: types.MimeTypeVP8, } From 3a0ee6690cda7c5d2eabd0272c9b5010210c954c Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 6 Sep 2023 18:01:35 -0700 Subject: [PATCH 17/20] working? --- go.mod | 1 + go.sum | 2 + pkg/config/pipeline.go | 1 - pkg/gstreamer/bin.go | 163 ++++++++++++-------- pkg/gstreamer/callbacks.go | 43 ++++-- pkg/gstreamer/pipeline.go | 87 +++++++---- pkg/gstreamer/state.go | 98 ++++++++++++ pkg/pipeline/builder/audio.go | 146 +++++++++++------- pkg/pipeline/builder/video.go | 271 ++++++++++++++++++++-------------- pkg/pipeline/controller.go | 92 ++++-------- pkg/pipeline/source/sdk.go | 111 +++++++------- pkg/pipeline/source/source.go | 1 + pkg/pipeline/source/web.go | 4 + pkg/pipeline/watch.go | 13 +- test/integration.go | 8 +- test/participant.go | 3 +- 16 files changed, 651 insertions(+), 393 deletions(-) create mode 100644 pkg/gstreamer/state.go diff --git a/go.mod b/go.mod index 630abddb..379324fe 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/nats-io/nats.go v1.28.0 // indirect github.com/nats-io/nkeys v0.4.4 // indirect github.com/nats-io/nuid v1.0.1 // indirect + github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.7 // indirect github.com/pion/ice/v2 v2.3.10 // indirect diff --git a/go.sum b/go.sum index 257cbe5c..10089e19 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= diff --git a/pkg/config/pipeline.go b/pkg/config/pipeline.go index 639de545..31bceb81 100644 --- a/pkg/config/pipeline.go +++ b/pkg/config/pipeline.go @@ -98,7 +98,6 @@ type TrackSource struct { MimeType types.MimeType PayloadType webrtc.PayloadType ClockRate uint32 - EOSFunc func() } type AudioConfig struct { diff --git a/pkg/gstreamer/bin.go b/pkg/gstreamer/bin.go index 1f9abaa2..32cb8e6f 100644 --- a/pkg/gstreamer/bin.go +++ b/pkg/gstreamer/bin.go @@ -17,6 +17,7 @@ package gstreamer import ( "fmt" "sync" + "time" "github.com/tinyzimmer/go-gst/gst" @@ -27,6 +28,7 @@ import ( // Bins are designed to hold a single stream, with any number of sources and sinks type Bin struct { *Callbacks + *StateManager pipeline *gst.Pipeline mu sync.Mutex @@ -34,7 +36,7 @@ type Bin struct { latency uint64 linkFunc func() error - eosFunc func() + eosFunc func() bool getSrcPad func(string) *gst.Pad getSinkPad func(string) *gst.Pad @@ -48,24 +50,35 @@ type Bin struct { func (b *Bin) NewBin(name string) *Bin { return &Bin{ - Callbacks: b.Callbacks, - pipeline: b.pipeline, - bin: gst.NewBin(name), - pads: make(map[string]*gst.GhostPad), + Callbacks: b.Callbacks, + StateManager: b.StateManager, + pipeline: b.pipeline, + bin: gst.NewBin(name), + pads: make(map[string]*gst.GhostPad), } } // Add src as a source of b. This should only be called once for each source bin func (b *Bin) AddSourceBin(src *Bin) error { + logger.Debugw(fmt.Sprintf("adding src %s to %s", src.bin.GetName(), b.bin.GetName())) return b.addBin(src, gst.PadDirectionSource) } // Add src as a sink of b. This should only be called once for each sink bin func (b *Bin) AddSinkBin(sink *Bin) error { + logger.Debugw(fmt.Sprintf("adding sink %s to %s", sink.bin.GetName(), b.bin.GetName())) return b.addBin(sink, gst.PadDirectionSink) } func (b *Bin) addBin(bin *Bin, direction gst.PadDirection) error { + b.LockStateShared() + defer b.UnlockStateShared() + + state := b.GetStateLocked() + if state > StateRunning { + return nil + } + b.mu.Lock() defer b.mu.Unlock() @@ -87,22 +100,24 @@ func (b *Bin) addBin(bin *Bin, direction gst.PadDirection) error { return errors.ErrGstPipelineError(err) } - if b.bin.GetState() == gst.StatePlaying { - if err := bin.link(); err != nil { - return err - } + if state == StateBuilding { + return nil + } - var err error - bin.mu.Lock() - if direction == gst.PadDirectionSource { - err = linkPeersLocked(bin, b) - } else { - err = linkPeersLocked(b, bin) - } - bin.mu.Unlock() - if err != nil { - return err - } + if err := bin.link(); err != nil { + return err + } + + var err error + bin.mu.Lock() + if direction == gst.PadDirectionSource { + err = linkPeersLocked(bin, b) + } else { + err = linkPeersLocked(b, bin) + } + bin.mu.Unlock() + if err != nil { + return err } return nil @@ -134,6 +149,9 @@ func (b *Bin) AddElements(elements ...*gst.Element) error { } func (b *Bin) RemoveSourceBin(name string) (bool, error) { + b.LockStateShared() + defer b.UnlockStateShared() + b.mu.Lock() defer b.mu.Unlock() @@ -144,41 +162,46 @@ func (b *Bin) RemoveSourceBin(name string) (bool, error) { b.srcs = append(b.srcs[:i], b.srcs[i+1:]...) break } - removed, err := s.RemoveSourceBin(name) - if removed || err != nil { - return removed, err - } } if src == nil { return false, nil } - if b.bin.GetState() != gst.StatePlaying { - if err := b.pipeline.Remove(src.bin.Element); err != nil { - return false, errors.ErrGstPipelineError(err) - } + state := b.GetStateLocked() + if state > StateRunning { return true, nil } - if err := src.bin.SetState(gst.StateNull); err != nil { - return false, err - } + if state != StateBuilding { + logger.Debugw(fmt.Sprintf("adding probe to %s", src.bin.GetName())) + src.mu.Lock() + srcGhostPad, sinkGhostPad := getGhostPads(src, b) + src.mu.Unlock() + + srcGhostPad.AddProbe(gst.PadProbeTypeBlocking, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { + logger.Debugw("probe running") + sinkPad := sinkGhostPad.GetTarget() + b.elements[0].ReleaseRequestPad(sinkPad) - src.mu.Lock() - srcPad, sinkPad := getGhostPads(src, b) - src.mu.Unlock() + srcGhostPad.Unlink(sinkGhostPad.Pad) + b.bin.RemovePad(sinkGhostPad.Pad) - srcPad.Unlink(sinkPad.Pad) - if err := b.pipeline.Remove(src.bin.Element); err != nil { - return false, errors.ErrGstPipelineError(err) + if err := b.pipeline.Remove(src.bin.Element); err != nil { + b.OnError(err) + } + + _ = src.bin.SetState(gst.StateNull) + return gst.PadProbeRemove + }) } - b.bin.RemovePad(sinkPad.Pad) - b.elements[0].ReleaseRequestPad(sinkPad.GetTarget()) return true, nil } func (b *Bin) RemoveSinkBin(name string) (bool, error) { + b.LockStateShared() + defer b.UnlockStateShared() + b.mu.Lock() defer b.mu.Unlock() @@ -189,16 +212,18 @@ func (b *Bin) RemoveSinkBin(name string) (bool, error) { b.sinks = append(b.sinks[:i], b.sinks[i+1:]...) break } - removed, err := s.RemoveSinkBin(name) - if removed || err != nil { - return removed, err - } } if sink == nil { return false, nil } - if b.bin.GetState() != gst.StatePlaying { + state := b.GetStateLocked() + if state > StateRunning { + return true, nil + } + + defer logger.Debugw(fmt.Sprintf("%s removed", name)) + if state == StateBuilding { if err := b.pipeline.Remove(sink.bin.Element); err != nil { return false, errors.ErrGstPipelineError(err) } @@ -221,7 +246,7 @@ func (b *Bin) RemoveSinkBin(name string) (bool, error) { return gst.PadProbeRemove } - if err = sink.bin.SetState(gst.StateNull); err != nil { + if err = sink.SetState(gst.StateNull); err != nil { logger.Warnw(fmt.Sprintf("failed to change %s state", sink.bin.GetName()), err) } @@ -234,7 +259,19 @@ func (b *Bin) RemoveSinkBin(name string) (bool, error) { } func (b *Bin) SetState(state gst.State) error { - return b.bin.SetState(state) + stateErr := make(chan error, 1) + go func() { + stateErr <- b.bin.SetState(state) + }() + select { + case <-time.After(stateChangeTimeout): + return errors.ErrPipelineFrozen + case err := <-stateErr: + if err != nil { + return errors.ErrGstPipelineError(err) + } + } + return nil } // Set a custom linking function for this bin's elements (used when you need to modify chain functions) @@ -261,8 +298,8 @@ func (b *Bin) SetGetSinkPad(f func(sinkName string) *gst.Pad) { b.getSinkPad = f } -// Set a custom EOS function (used for appsrc) -func (b *Bin) SetEOSFunc(f func()) { +// Set a custom EOS function (used for appsrc, input-selector). If it returns true, EOS will also be sent to src bins +func (b *Bin) SetEOSFunc(f func() bool) { b.mu.Lock() defer b.mu.Unlock() @@ -354,19 +391,19 @@ func linkPeersLocked(src, sink *Bin) error { sinkState := sink.bin.GetState() if srcState != sinkState { - if sinkState == gst.StatePlaying { + if srcState == gst.StateNull { srcPad.AddProbe(gst.PadProbeTypeBlockDownstream, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { if padReturn := srcPad.Link(sinkPad.Pad); padReturn != gst.PadLinkOK { logger.Errorw("failed to link", errors.ErrPadLinkFailed(src.bin.GetName(), sink.bin.GetName(), padReturn.String())) } return gst.PadProbeRemove }) - return src.bin.SetState(gst.StatePlaying) + return src.SetState(gst.StatePlaying) } - if srcState == gst.StatePlaying { + if sinkState == gst.StateNull { srcPad.AddProbe(gst.PadProbeTypeBlockDownstream, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { - if err = sink.bin.SetState(gst.StatePlaying); err != nil { + if err = sink.SetState(gst.StatePlaying); err != nil { src.OnError(errors.ErrGstPipelineError(err)) return gst.PadProbeUnhandled } @@ -410,14 +447,24 @@ func (b *Bin) linkPeersWithQueueLocked(src, sink *Bin) error { func (b *Bin) sendEOS() { b.mu.Lock() - defer b.mu.Unlock() + eosFunc := b.eosFunc + srcs := b.srcs + b.mu.Unlock() - if b.eosFunc != nil { - b.eosFunc() - } else if len(b.srcs) > 0 { - for _, src := range b.srcs { - src.sendEOS() + if eosFunc != nil && !eosFunc() { + return + } + + if len(srcs) > 0 { + var wg sync.WaitGroup + wg.Add(len(b.srcs)) + for _, src := range srcs { + go func(s *Bin) { + s.sendEOS() + wg.Done() + }(src) } + wg.Wait() } else if len(b.elements) > 0 { b.bin.SendEvent(gst.NewEOSEvent()) } diff --git a/pkg/gstreamer/callbacks.go b/pkg/gstreamer/callbacks.go index cff14d4b..9ed44415 100644 --- a/pkg/gstreamer/callbacks.go +++ b/pkg/gstreamer/callbacks.go @@ -53,6 +53,7 @@ func (c *Callbacks) OnError(err error) { c.mu.RLock() onError := c.onError c.mu.RUnlock() + if onError != nil { onError(err) } @@ -65,12 +66,14 @@ func (c *Callbacks) AddOnStop(f func() error) { } func (c *Callbacks) OnStop() error { - errArray := &errors.ErrArray{} c.mu.RLock() - for _, onStop := range c.onStop { - errArray.Check(onStop()) - } + onStop := c.onStop c.mu.RUnlock() + + errArray := &errors.ErrArray{} + for _, f := range onStop { + errArray.Check(f()) + } return errArray.ToError() } @@ -82,10 +85,12 @@ func (c *Callbacks) AddOnTrackAdded(f func(*config.TrackSource)) { func (c *Callbacks) OnTrackAdded(ts *config.TrackSource) { c.mu.RLock() - for _, onTrackAdded := range c.onTrackAdded { - onTrackAdded(ts) - } + onTrackAdded := c.onTrackAdded c.mu.RUnlock() + + for _, f := range onTrackAdded { + f(ts) + } } func (c *Callbacks) AddOnTrackMuted(f func(string)) { @@ -96,10 +101,12 @@ func (c *Callbacks) AddOnTrackMuted(f func(string)) { func (c *Callbacks) OnTrackMuted(trackID string) { c.mu.RLock() - for _, onTrackMuted := range c.onTrackMuted { - onTrackMuted(trackID) - } + onTrackMuted := c.onTrackMuted c.mu.RUnlock() + + for _, f := range onTrackMuted { + f(trackID) + } } func (c *Callbacks) AddOnTrackUnmuted(f func(string, time.Duration)) { @@ -110,10 +117,12 @@ func (c *Callbacks) AddOnTrackUnmuted(f func(string, time.Duration)) { func (c *Callbacks) OnTrackUnmuted(trackID string, pts time.Duration) { c.mu.RLock() - for _, onTrackUnmuted := range c.onTrackUnmuted { - onTrackUnmuted(trackID, pts) - } + onTrackUnmuted := c.onTrackUnmuted c.mu.RUnlock() + + for _, f := range onTrackUnmuted { + f(trackID, pts) + } } func (c *Callbacks) AddOnTrackRemoved(f func(string)) { @@ -124,8 +133,10 @@ func (c *Callbacks) AddOnTrackRemoved(f func(string)) { func (c *Callbacks) OnTrackRemoved(trackID string) { c.mu.RLock() - for _, onTrackRemoved := range c.onTrackRemoved { - onTrackRemoved(trackID) - } + onTrackRemoved := c.onTrackRemoved c.mu.RUnlock() + + for _, f := range onTrackRemoved { + f(trackID) + } } diff --git a/pkg/gstreamer/pipeline.go b/pkg/gstreamer/pipeline.go index 3f924b64..9d5f9a74 100644 --- a/pkg/gstreamer/pipeline.go +++ b/pkg/gstreamer/pipeline.go @@ -15,6 +15,8 @@ package gstreamer import ( + "time" + "github.com/frostbyte73/core" "github.com/tinyzimmer/go-glib/glib" "github.com/tinyzimmer/go-gst/gst" @@ -23,14 +25,17 @@ import ( "github.com/livekit/protocol/logger" ) +const ( + stateChangeTimeout = time.Second * 15 + stopTimeout = time.Second * 30 +) + type Pipeline struct { *Bin - loop *glib.MainLoop - + loop *glib.MainLoop binsAdded bool elementsAdded bool - started core.Fuse running chan struct{} stopped core.Fuse } @@ -45,14 +50,14 @@ func NewPipeline(name string, latency uint64, callbacks *Callbacks) (*Pipeline, return &Pipeline{ Bin: &Bin{ - Callbacks: callbacks, - pipeline: pipeline, - bin: pipeline.Bin, - latency: latency, - queues: make(map[string]*gst.Element), + Callbacks: callbacks, + StateManager: &StateManager{}, + pipeline: pipeline, + bin: pipeline.Bin, + latency: latency, + queues: make(map[string]*gst.Element), }, loop: glib.NewMainLoop(glib.MainContextDefault(), false), - started: core.NewFuse(), running: make(chan struct{}), stopped: core.NewFuse(), }, nil @@ -102,9 +107,11 @@ func (p *Pipeline) SetState(state gst.State) error { p.mu.Lock() defer p.mu.Unlock() - if err := p.pipeline.SetState(state); err != nil { - return errors.ErrGstPipelineError(err) - } + stateErr := make(chan error, 1) + go func() { + stateErr <- p.pipeline.SetState(state) + }() + if state == gst.StateNull { for _, src := range p.srcs { if err := src.SetState(gst.StateNull); err != nil { @@ -112,19 +119,29 @@ func (p *Pipeline) SetState(state gst.State) error { } } } + + select { + case <-time.After(stateChangeTimeout): + return errors.ErrPipelineFrozen + case err := <-stateErr: + if err != nil { + return errors.ErrGstPipelineError(err) + } + } + return nil } func (p *Pipeline) Run() error { - p.started.Once(func() { + if _, ok := p.UpgradeState(StateStarted); ok { if err := p.SetState(gst.StatePlaying); err != nil { - p.OnError(err) - return + return err + } + if _, ok = p.UpgradeState(StateRunning); ok { + p.loop.Run() } - logger.Debugw("starting main loop") - p.loop.Run() close(p.running) - }) + } // wait <-p.running @@ -132,18 +149,36 @@ func (p *Pipeline) Run() error { } func (p *Pipeline) SendEOS() { - p.sendEOS() + old, ok := p.UpgradeState(StateEOS) + if ok { + if old >= StateRunning { + p.sendEOS() + } else { + p.Stop() + } + } } func (p *Pipeline) Stop() { - p.stopped.Once(func() { - defer p.loop.Quit() + old, ok := p.UpgradeState(StateStopping) + if !ok { + return + } - _ = p.SetState(gst.StateNull) - if err := p.OnStop(); err != nil { - p.OnError(err) - } - }) + if err := p.OnStop(); err != nil { + logger.Errorw("onStop failure", err) + p.OnError(err) + } + + if old >= StateRunning { + p.loop.Quit() + logger.Debugw("main loop closed") + } + + p.UpgradeState(StateFinished) + go func() { + _ = p.pipeline.SetState(gst.StateNull) + }() } func (p *Pipeline) DebugBinToDotData(details gst.DebugGraphDetails) string { diff --git a/pkg/gstreamer/state.go b/pkg/gstreamer/state.go new file mode 100644 index 00000000..e413e38d --- /dev/null +++ b/pkg/gstreamer/state.go @@ -0,0 +1,98 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gstreamer + +import ( + "fmt" + "sync" + + "github.com/livekit/protocol/logger" +) + +type State int + +const ( + StateBuilding State = iota + StateStarted + StateRunning + StateEOS + StateStopping + StateFinished +) + +func (s State) String() string { + switch s { + case StateBuilding: + return "building" + case StateStarted: + return "starting" + case StateRunning: + return "running" + case StateEOS: + return "eos" + case StateStopping: + return "stopping" + case StateFinished: + return "finished" + default: + return "unknown" + } +} + +type StateManager struct { + lock sync.RWMutex + state State +} + +func (s *StateManager) GetState() State { + s.lock.RLock() + defer s.lock.RUnlock() + + return s.state +} + +func (s *StateManager) GetStateLocked() State { + return s.state +} + +func (s *StateManager) LockState() { + s.lock.Lock() +} + +func (s *StateManager) UnlockState() { + s.lock.Unlock() +} + +func (s *StateManager) LockStateShared() { + s.lock.RLock() +} + +func (s *StateManager) UnlockStateShared() { + s.lock.RUnlock() +} + +func (s *StateManager) UpgradeState(state State) (State, bool) { + s.lock.Lock() + defer s.lock.Unlock() + + old := s.state + if old >= state { + return old, false + } else { + logger.Debugw(fmt.Sprintf("pipeline state %v -> %v", old, state)) + s.state = state + return old, true + } +} diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index db498856..19cf1dd6 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -16,6 +16,7 @@ package builder import ( "fmt" + "sync" "github.com/tinyzimmer/go-gst/gst" @@ -23,97 +24,80 @@ import ( "github.com/livekit/egress/pkg/errors" "github.com/livekit/egress/pkg/gstreamer" "github.com/livekit/egress/pkg/types" + lksdk "github.com/livekit/server-sdk-go" ) const audioMixerLatency = uint64(2e9) type AudioBin struct { bin *gstreamer.Bin + p *config.PipelineConfig + + mu sync.Mutex + tracks map[string]struct{} } -func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*AudioBin, error) { +func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error { b := &AudioBin{ - bin: pipeline.NewBin("audio"), + bin: pipeline.NewBin("audio"), + tracks: make(map[string]struct{}), } switch p.SourceType { case types.SourceTypeWeb: if err := b.buildWebInput(p); err != nil { - return nil, err + return err } case types.SourceTypeSDK: if err := b.buildSDKInput(p); err != nil { - return nil, err + return err } + + pipeline.AddOnTrackAdded(b.onTrackAdded) + pipeline.AddOnTrackRemoved(b.onTrackRemoved) } if len(p.Outputs) > 1 { tee, err := gst.NewElementWithName("tee", "audio_tee") if err != nil { - return nil, err + return err } - if err = b.bin.AddElement(tee); err != nil { - return nil, err + return err } } - if err := pipeline.AddSourceBin(b.bin); err != nil { - return nil, err - } - - return b, nil + return pipeline.AddSourceBin(b.bin) } -func (b *AudioBin) AddAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { - appSrcBin := b.bin.NewBin(ts.TrackID) - appSrcBin.SetEOSFunc(ts.EOSFunc) - - ts.AppSrc.Element.SetArg("format", "time") - if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { - return err - } - if err := appSrcBin.AddElement(ts.AppSrc.Element); err != nil { - return err +func (b *AudioBin) onTrackAdded(ts *config.TrackSource) { + if b.bin.GetState() > gstreamer.StateRunning { + return } - switch ts.MimeType { - case types.MimeTypeOpus: - if err := ts.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( - "application/x-rtp,media=audio,payload=%d,encoding-name=OPUS,clock-rate=%d", - ts.PayloadType, ts.ClockRate, - ))); err != nil { - return errors.ErrGstPipelineError(err) - } - - rtpOpusDepay, err := gst.NewElement("rtpopusdepay") - if err != nil { - return errors.ErrGstPipelineError(err) - } - - opusDec, err := gst.NewElement("opusdec") - if err != nil { - return errors.ErrGstPipelineError(err) - } - - if err = appSrcBin.AddElements(rtpOpusDepay, opusDec); err != nil { - return err + if ts.Kind == lksdk.TrackKindAudio { + if err := b.addAudioAppSrcBin(b.p, ts); err != nil { + b.bin.OnError(err) } - - default: - return errors.ErrNotSupported(string(ts.MimeType)) } +} - if err := addAudioConverter(appSrcBin, p); err != nil { - return err +func (b *AudioBin) onTrackRemoved(trackID string) { + if b.bin.GetState() > gstreamer.StateRunning { + return } - if err := b.bin.AddSourceBin(appSrcBin); err != nil { - return err - } + b.mu.Lock() + _, ok := b.tracks[trackID] + delete(b.tracks, trackID) + b.mu.Unlock() - return nil + if ok { + if _, err := b.bin.RemoveSourceBin(trackID); err != nil { + b.bin.OnError(err) + } + } } func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { @@ -142,7 +126,7 @@ func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { if p.AudioTrack != nil { - if err := b.AddAudioAppSrcBin(p, p.AudioTrack); err != nil { + if err := b.addAudioAppSrcBin(p, p.AudioTrack); err != nil { return err } } @@ -161,6 +145,62 @@ func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { return nil } +func (b *AudioBin) addAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { + b.mu.Lock() + defer b.mu.Unlock() + + b.tracks[ts.TrackID] = struct{}{} + + appSrcBin := b.bin.NewBin(ts.TrackID) + appSrcBin.SetEOSFunc(func() bool { + return false + }) + ts.AppSrc.Element.SetArg("format", "time") + if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { + return err + } + if err := appSrcBin.AddElement(ts.AppSrc.Element); err != nil { + return err + } + + switch ts.MimeType { + case types.MimeTypeOpus: + if err := ts.AppSrc.Element.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( + "application/x-rtp,media=audio,payload=%d,encoding-name=OPUS,clock-rate=%d", + ts.PayloadType, ts.ClockRate, + ))); err != nil { + return errors.ErrGstPipelineError(err) + } + + rtpOpusDepay, err := gst.NewElement("rtpopusdepay") + if err != nil { + return errors.ErrGstPipelineError(err) + } + + opusDec, err := gst.NewElement("opusdec") + if err != nil { + return errors.ErrGstPipelineError(err) + } + + if err = appSrcBin.AddElements(rtpOpusDepay, opusDec); err != nil { + return err + } + + default: + return errors.ErrNotSupported(string(ts.MimeType)) + } + + if err := addAudioConverter(appSrcBin, p); err != nil { + return err + } + + if err := b.bin.AddSourceBin(appSrcBin); err != nil { + return err + } + + return nil +} + func (b *AudioBin) addAudioTestSrcBin(p *config.PipelineConfig) error { testSrcBin := b.bin.NewBin("audio_test_src") if err := b.bin.AddSourceBin(testSrcBin); err != nil { diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index 55b4d7ea..2a9bf036 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -29,12 +29,14 @@ import ( "github.com/livekit/egress/pkg/gstreamer" "github.com/livekit/egress/pkg/types" "github.com/livekit/protocol/logger" + lksdk "github.com/livekit/server-sdk-go" ) const videoTestSrcName = "video_test_src" type VideoBin struct { bin *gstreamer.Bin + p *config.PipelineConfig lastPTS atomic.Duration nextPTS atomic.Duration @@ -46,80 +48,97 @@ type VideoBin struct { selector *gst.Element } -func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) (*VideoBin, error) { +func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error { b := &VideoBin{ bin: pipeline.NewBin("video"), + p: p, } switch p.SourceType { case types.SourceTypeWeb: if err := b.buildWebInput(p); err != nil { - return nil, err + return err } case types.SourceTypeSDK: if err := b.buildSDKInput(p); err != nil { - return nil, err + return err } + + pipeline.AddOnTrackAdded(b.onTrackAdded) + pipeline.AddOnTrackRemoved(b.onTrackRemoved) + pipeline.AddOnTrackMuted(b.onTrackMuted) + pipeline.AddOnTrackUnmuted(b.onTrackUnmuted) } if len(p.Outputs) > 1 { tee, err := gst.NewElementWithName("tee", "video_tee") if err != nil { - return nil, err + return err } if err = b.bin.AddElement(tee); err != nil { - return nil, err + return err } } - if err := pipeline.AddSourceBin(b.bin); err != nil { - return nil, err - } - - return b, nil + return pipeline.AddSourceBin(b.bin) } -func (b *VideoBin) AddVideoAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { - appSrcBin, err := buildVideoAppSrcBin(b.bin, p, ts) - if err != nil { - return err +func (b *VideoBin) onTrackAdded(ts *config.TrackSource) { + if b.bin.GetState() > gstreamer.StateRunning { + return } - if p.VideoTranscoding { - b.createSrcPad(ts.TrackID) + if ts.Kind == lksdk.TrackKindVideo { + if err := b.addAppSrcBin(b.p, ts); err != nil { + b.bin.OnError(err) + } } +} - if err = b.bin.AddSourceBin(appSrcBin); err != nil { - return err +func (b *VideoBin) onTrackRemoved(trackID string) { + if b.bin.GetState() > gstreamer.StateRunning { + return } - if p.VideoTranscoding { - return b.setSelectorPad(ts.TrackID) + b.mu.Lock() + pad := b.pads[trackID] + if pad == nil { + b.mu.Unlock() + return } + delete(b.pads, trackID) + b.mu.Unlock() - return nil -} - -func (b *VideoBin) getSrcPad(name string) *gst.Pad { - b.mu.Lock() - defer b.mu.Unlock() + if b.selectedPad == trackID { + if err := b.setSelectorPad(videoTestSrcName); err != nil { + b.bin.OnError(err) + } + } - return b.pads[name] + if _, err := b.bin.RemoveSourceBin(trackID); err != nil { + b.bin.OnError(err) + } } func (b *VideoBin) onTrackMuted(trackID string) { - if b.selectedPad != trackID { + if b.bin.GetState() > gstreamer.StateRunning { return } - if err := b.setSelectorPad(videoTestSrcName); err != nil { - logger.Errorw("failed to set selector pad", err) + if b.selectedPad == trackID { + if err := b.setSelectorPad(videoTestSrcName); err != nil { + logger.Errorw("failed to set selector pad", err) + } } } func (b *VideoBin) onTrackUnmuted(trackID string, pts time.Duration) { + if b.bin.GetState() > gstreamer.StateRunning { + return + } + b.mu.Lock() defer b.mu.Unlock() @@ -131,75 +150,6 @@ func (b *VideoBin) onTrackUnmuted(trackID string, pts time.Duration) { b.nextPad = trackID } -func (b *VideoBin) createSrcPad(trackID string) { - b.mu.Lock() - defer b.mu.Unlock() - - pad := b.selector.GetRequestPad("sink_%u") - pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - buffer := info.GetBuffer() - for b.nextPTS.Load() != 0 { - time.Sleep(time.Millisecond * 100) - } - if buffer.PresentationTimestamp() < b.lastPTS.Load() { - return gst.PadProbeDrop - } - b.lastPTS.Store(buffer.PresentationTimestamp()) - return gst.PadProbeOK - }) - b.pads[trackID] = pad -} - -func (b *VideoBin) createTestSrcPad() { - b.mu.Lock() - defer b.mu.Unlock() - - pad := b.selector.GetRequestPad("sink_%u") - pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { - buffer := info.GetBuffer() - if buffer.PresentationTimestamp() < b.lastPTS.Load() { - return gst.PadProbeDrop - } - if nextPTS := b.nextPTS.Load(); nextPTS != 0 && buffer.PresentationTimestamp() >= nextPTS { - if err := b.setSelectorPad(b.nextPad); err != nil { - logger.Errorw("failed to unmute", err) - return gst.PadProbeDrop - } - b.nextPad = "" - b.nextPTS.Store(0) - } - b.lastPTS.Store(buffer.PresentationTimestamp()) - return gst.PadProbeOK - }) - b.pads[videoTestSrcName] = pad -} - -// TODO: go-gst should accept objects directly and handle conversion to C -func (b *VideoBin) setSelectorPad(name string) error { - b.mu.Lock() - defer b.mu.Unlock() - - pad := b.pads[name] - - pt, err := b.selector.GetPropertyType("active-pad") - if err != nil { - return errors.ErrGstPipelineError(err) - } - - val, err := glib.ValueInit(pt) - if err != nil { - return errors.ErrGstPipelineError(err) - } - val.SetInstance(uintptr(unsafe.Pointer(pad.Instance()))) - - if err = b.selector.SetPropertyValue("active-pad", val); err != nil { - return errors.ErrGstPipelineError(err) - } - - b.selectedPad = name - return nil -} - func (b *VideoBin) buildWebInput(p *config.PipelineConfig) error { xImageSrc, err := gst.NewElement("ximagesrc") if err != nil { @@ -260,26 +210,33 @@ func (b *VideoBin) buildSDKInput(p *config.PipelineConfig) error { } if p.VideoTrack != nil { - if err := b.AddVideoAppSrcBin(p, p.VideoTrack); err != nil { + if err := b.addAppSrcBin(p, p.VideoTrack); err != nil { return err } } if p.VideoTranscoding { + b.bin.SetGetSrcPad(b.getSrcPad) + b.bin.SetEOSFunc(func() bool { + b.mu.Lock() + selected := b.selectedPad + pad := b.pads[videoTestSrcName] + b.mu.Unlock() + + if selected == videoTestSrcName { + pad.SendEvent(gst.NewEOSEvent()) + } + return false + }) + if err := b.addVideoTestSrcBin(p); err != nil { return err } - if p.VideoTrack == nil { if err := b.setSelectorPad(videoTestSrcName); err != nil { return err } } - - b.bin.SetGetSrcPad(b.getSrcPad) - b.bin.Callbacks.AddOnTrackMuted(b.onTrackMuted) - b.bin.Callbacks.AddOnTrackUnmuted(b.onTrackUnmuted) - if err := b.addEncoder(p); err != nil { return err } @@ -288,10 +245,29 @@ func (b *VideoBin) buildSDKInput(p *config.PipelineConfig) error { return nil } +func (b *VideoBin) addAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { + appSrcBin, err := buildVideoAppSrcBin(b.bin, p, ts) + if err != nil { + return err + } + + if p.VideoTranscoding { + b.createSrcPad(ts.TrackID) + } + + if err = b.bin.AddSourceBin(appSrcBin); err != nil { + return err + } + + if p.VideoTranscoding { + return b.setSelectorPad(ts.TrackID) + } + + return nil +} + func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) (*gstreamer.Bin, error) { b := videoBin.NewBin(ts.TrackID) - b.SetEOSFunc(ts.EOSFunc) - ts.AppSrc.Element.SetArg("format", "time") if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { return nil, errors.ErrGstPipelineError(err) @@ -469,6 +445,7 @@ func (b *VideoBin) addSelector(p *config.PipelineConfig) error { if err != nil { return errors.ErrGstPipelineError(err) } + videoRate, err := gst.NewElement("videorate") if err != nil { return errors.ErrGstPipelineError(err) @@ -640,3 +617,81 @@ func newVideoCapsFilter(p *config.PipelineConfig, includeFramerate bool) (*gst.E } return caps, nil } + +func (b *VideoBin) getSrcPad(name string) *gst.Pad { + b.mu.Lock() + defer b.mu.Unlock() + + return b.pads[name] +} + +func (b *VideoBin) createSrcPad(trackID string) { + b.mu.Lock() + defer b.mu.Unlock() + + pad := b.selector.GetRequestPad("sink_%u") + pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + buffer := info.GetBuffer() + for b.nextPTS.Load() != 0 { + time.Sleep(time.Millisecond * 100) + } + if buffer.PresentationTimestamp() < b.lastPTS.Load() { + return gst.PadProbeDrop + } + b.lastPTS.Store(buffer.PresentationTimestamp()) + return gst.PadProbeOK + }) + b.pads[trackID] = pad +} + +func (b *VideoBin) createTestSrcPad() { + b.mu.Lock() + defer b.mu.Unlock() + + pad := b.selector.GetRequestPad("sink_%u") + pad.AddProbe(gst.PadProbeTypeBuffer, func(pad *gst.Pad, info *gst.PadProbeInfo) gst.PadProbeReturn { + buffer := info.GetBuffer() + if buffer.PresentationTimestamp() < b.lastPTS.Load() { + return gst.PadProbeDrop + } + if nextPTS := b.nextPTS.Load(); nextPTS != 0 && buffer.PresentationTimestamp() >= nextPTS { + if err := b.setSelectorPad(b.nextPad); err != nil { + logger.Errorw("failed to unmute", err) + return gst.PadProbeDrop + } + b.nextPad = "" + b.nextPTS.Store(0) + } + if b.selectedPad == videoTestSrcName { + b.lastPTS.Store(buffer.PresentationTimestamp()) + } + return gst.PadProbeOK + }) + b.pads[videoTestSrcName] = pad +} + +// TODO: go-gst should accept objects directly and handle conversion to C +func (b *VideoBin) setSelectorPad(name string) error { + b.mu.Lock() + defer b.mu.Unlock() + + pad := b.pads[name] + + pt, err := b.selector.GetPropertyType("active-pad") + if err != nil { + return errors.ErrGstPipelineError(err) + } + + val, err := glib.ValueInit(pt) + if err != nil { + return errors.ErrGstPipelineError(err) + } + val.SetInstance(uintptr(unsafe.Pointer(pad.Instance()))) + + if err = b.selector.SetPropertyValue("active-pad", val); err != nil { + return errors.ErrGstPipelineError(err) + } + + b.selectedPad = name + return nil +} diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index ef444cc1..054e0b50 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -34,12 +34,10 @@ import ( "github.com/livekit/protocol/logger" "github.com/livekit/protocol/tracer" "github.com/livekit/protocol/utils" - lksdk "github.com/livekit/server-sdk-go" ) const ( pipelineName = "pipeline" - eosTimeout = time.Second * 30 ) type Controller struct { @@ -59,7 +57,6 @@ type Controller struct { playing core.Fuse eosSent core.Fuse stopped core.Fuse - eosTimer *time.Timer } func New(ctx context.Context, conf *config.PipelineConfig) (*Controller, error) { @@ -118,43 +115,28 @@ func (c *Controller) BuildPipeline() error { } p.SetWatch(c.messageWatch) - p.AddOnStop(c.OnStop) - - var audioBin *builder.AudioBin - var videoBin *builder.VideoBin + p.AddOnStop(func() error { + c.stopped.Break() + return nil + }) + if c.SourceType == types.SourceTypeSDK { + p.SetEOSFunc(func() bool { + c.src.(*source.SDKSource).CloseWriters() + return true + }) + } if c.AudioEnabled { - audioBin, err = builder.BuildAudioBin(p, c.PipelineConfig) - if err != nil { + if err = builder.BuildAudioBin(p, c.PipelineConfig); err != nil { return err } } if c.VideoEnabled { - videoBin, err = builder.BuildVideoBin(p, c.PipelineConfig) - if err != nil { + if err = builder.BuildVideoBin(p, c.PipelineConfig); err != nil { return err } } - p.AddOnTrackAdded(func(ts *config.TrackSource) { - switch ts.Kind { - case lksdk.TrackKindAudio: - if err := audioBin.AddAudioAppSrcBin(c.PipelineConfig, ts); err != nil { - p.OnError(err) - } - case lksdk.TrackKindVideo: - if err := videoBin.AddVideoAppSrcBin(c.PipelineConfig, ts); err != nil { - p.OnError(err) - } - } - }) - p.AddOnTrackRemoved(func(trackID string) { - _, err := p.RemoveSinkBin(trackID) - if err != nil { - p.OnError(err) - } - }) - for egressType := range c.Outputs { var sinkBin *gstreamer.Bin switch egressType { @@ -194,6 +176,12 @@ func (c *Controller) Run(ctx context.Context) *livekit.EgressInfo { c.Info.StartedAt = time.Now().UnixNano() defer func() { now := time.Now().UnixNano() + logger.Debugw("CLOSING") + + if c.SourceType == types.SourceTypeSDK { + c.updateDuration(c.src.GetEndedAt()) + } + c.Info.UpdatedAt = now c.Info.EndedAt = now @@ -457,6 +445,8 @@ func (c *Controller) SendEOS(ctx context.Context) { defer span.End() c.eosSent.Once(func() { + logger.Debugw("Sending EOS") + if c.limitTimer != nil { c.limitTimer.Stop() } @@ -483,24 +473,7 @@ func (c *Controller) SendEOS(ctx context.Context) { case livekit.EgressStatus_EGRESS_ENDING, livekit.EgressStatus_EGRESS_LIMIT_REACHED: - go func() { - logger.Infow("sending EOS to pipeline") - - c.eosTimer = time.AfterFunc(eosTimeout, func() { - logger.Errorw("pipeline frozen", nil, "stream", !c.FinalizationRequired) - if c.Debug.EnableProfiling { - c.uploadDebugFiles() - } - - if c.FinalizationRequired { - c.OnError(errors.ErrPipelineFrozen) - } else { - c.p.Stop() - } - }) - - c.p.SendEOS() - }() + go c.p.SendEOS() } switch c.src.(type) { @@ -511,26 +484,17 @@ func (c *Controller) SendEOS(ctx context.Context) { } func (c *Controller) OnError(err error) { - if c.Info.Error == "" && (!c.eosSent.IsBroken() || c.FinalizationRequired) { - c.Info.Error = err.Error() - } - go c.p.Stop() -} - -func (c *Controller) OnStop() error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.eosTimer != nil { - c.eosTimer.Stop() + if errors.Is(err, errors.ErrPipelineFrozen) { + if c.Debug.EnableProfiling { + c.uploadDebugFiles() + } } - switch c.src.(type) { - case *source.SDKSource: - c.updateDuration(c.src.GetEndedAt()) + if c.Info.Error == "" && (!c.eosSent.IsBroken() || c.FinalizationRequired) { + c.Info.Error = err.Error() } - return nil + go c.p.Stop() } func (c *Controller) updateDuration(endedAt int64) { diff --git a/pkg/pipeline/source/sdk.go b/pkg/pipeline/source/sdk.go index eb08b9e9..0ad3a80a 100644 --- a/pkg/pipeline/source/sdk.go +++ b/pkg/pipeline/source/sdk.go @@ -56,10 +56,9 @@ type SDKSource struct { filenameReplacements map[string]string errors chan error - active atomic.Int32 - audioWriter *sdk.AppWriter - videoWriter *sdk.AppWriter - closed core.Fuse + writers map[string]*sdk.AppWriter + active atomic.Int32 + closed core.Fuse startRecording chan struct{} endRecording chan struct{} @@ -78,6 +77,7 @@ func NewSDKSource(ctx context.Context, p *config.PipelineConfig, callbacks *gstr }), initialized: core.NewFuse(), filenameReplacements: make(map[string]string), + writers: make(map[string]*sdk.AppWriter), closed: core.NewFuse(), startRecording: startRecording, endRecording: make(chan struct{}), @@ -94,18 +94,22 @@ func (s *SDKSource) StartRecording() chan struct{} { return s.startRecording } -func (s *SDKSource) GetStartTime() int64 { - return s.sync.GetStartedAt() +func (s *SDKSource) EndRecording() chan struct{} { + return s.endRecording } func (s *SDKSource) Playing(trackID string) { - if w := s.getWriterForTrack(trackID); w != nil { - w.Play() + s.mu.Lock() + writer := s.writers[trackID] + s.mu.Unlock() + + if writer != nil { + writer.Play() } } -func (s *SDKSource) EndRecording() chan struct{} { - return s.endRecording +func (s *SDKSource) GetStartedAt() int64 { + return s.sync.GetStartedAt() } func (s *SDKSource) GetEndedAt() int64 { @@ -116,12 +120,17 @@ func (s *SDKSource) CloseWriters() { s.closed.Once(func() { s.sync.End() - if s.audioWriter != nil { - go s.audioWriter.Drain(false) - } - if s.videoWriter != nil { - go s.videoWriter.Drain(false) + var wg sync.WaitGroup + s.mu.Lock() + wg.Add(len(s.writers)) + for _, w := range s.writers { + go func(writer *sdk.AppWriter) { + defer wg.Done() + writer.Drain(false) + }(w) } + s.mu.Unlock() + wg.Wait() }) } @@ -327,8 +336,10 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo return } - ts.EOSFunc = s.CloseWriters - s.audioWriter = writer + s.mu.Lock() + s.writers[ts.TrackID] = writer + s.mu.Unlock() + if s.initialized.IsBroken() { s.callbacks.OnTrackAdded(ts) } else { @@ -351,8 +362,10 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo return } - ts.EOSFunc = s.CloseWriters - s.videoWriter = writer + s.mu.Lock() + s.writers[ts.TrackID] = writer + s.mu.Unlock() + if s.initialized.IsBroken() { s.callbacks.OnTrackAdded(ts) } else { @@ -381,6 +394,7 @@ func (s *SDKSource) onTrackSubscribed(track *webrtc.TrackRemote, pub *lksdk.Remo s.TrackKind = pub.Kind().String() if pub.Kind() == lksdk.TrackKindVideo && s.Outputs[types.EgressTypeWebsocket] != nil { onSubscribeErr = errors.ErrIncompatible("websocket", ts.MimeType) + s.mu.Unlock() return } s.TrackSource = strings.ToLower(pub.Source().String()) @@ -443,52 +457,45 @@ func (s *SDKSource) onTrackPublished(pub *lksdk.RemoteTrackPublication, rp *lksd } func (s *SDKSource) onTrackMuted(pub lksdk.TrackPublication, _ lksdk.Participant) { - if w := s.getWriterForTrack(pub.SID()); w != nil { - w.SetTrackMuted(true) + s.mu.Lock() + writer := s.writers[pub.SID()] + s.mu.Unlock() + + if writer != nil { + writer.SetTrackMuted(true) } } func (s *SDKSource) onTrackUnmuted(pub lksdk.TrackPublication, _ lksdk.Participant) { - if w := s.getWriterForTrack(pub.SID()); w != nil { - w.SetTrackMuted(false) - } -} + s.mu.Lock() + writer := s.writers[pub.SID()] + s.mu.Unlock() -func (s *SDKSource) getWriterForTrack(trackID string) *sdk.AppWriter { - if s.audioWriter != nil && s.audioWriter.TrackID() == trackID { - return s.audioWriter - } - if s.videoWriter != nil && s.videoWriter.TrackID() == trackID { - return s.videoWriter + if writer != nil { + writer.SetTrackMuted(false) } - return nil } func (s *SDKSource) onTrackUnsubscribed(_ *webrtc.TrackRemote, pub *lksdk.RemoteTrackPublication, _ *lksdk.RemoteParticipant) { + logger.Debugw("track unsubscribed", "trackID", pub.SID()) s.onTrackFinished(pub.SID()) } func (s *SDKSource) onTrackFinished(trackID string) { - var w *sdk.AppWriter - if s.audioWriter != nil && s.audioWriter.TrackID() == trackID { - logger.Infow("removing audio writer") - w = s.audioWriter - s.audioWriter = nil - } else if s.videoWriter != nil && s.videoWriter.TrackID() == trackID { - logger.Infow("removing video writer") - w = s.videoWriter - s.videoWriter = nil - } else { - return - } - - w.Drain(true) - active := s.active.Dec() - if s.RequestType == types.RequestTypeParticipant { - s.callbacks.OnTrackRemoved(trackID) - s.sync.RemoveTrack(trackID) - } else if active == 0 { - s.onDisconnected() + s.mu.Lock() + writer := s.writers[trackID] + delete(s.writers, trackID) + s.mu.Unlock() + + if writer != nil { + writer.Drain(true) + active := s.active.Dec() + if s.RequestType == types.RequestTypeParticipant { + s.callbacks.OnTrackRemoved(trackID) + s.sync.RemoveTrack(trackID) + } else if active == 0 { + s.onDisconnected() + } } } diff --git a/pkg/pipeline/source/source.go b/pkg/pipeline/source/source.go index b4176bf1..79be65d8 100644 --- a/pkg/pipeline/source/source.go +++ b/pkg/pipeline/source/source.go @@ -26,6 +26,7 @@ import ( type Source interface { StartRecording() chan struct{} EndRecording() chan struct{} + GetStartedAt() int64 GetEndedAt() int64 Close() } diff --git a/pkg/pipeline/source/web.go b/pkg/pipeline/source/web.go index 8d47f675..6385ca2a 100644 --- a/pkg/pipeline/source/web.go +++ b/pkg/pipeline/source/web.go @@ -95,6 +95,10 @@ func (s *WebSource) EndRecording() chan struct{} { return s.endRecording } +func (s *WebSource) GetStartedAt() int64 { + return time.Now().UnixNano() +} + func (s *WebSource) GetEndedAt() int64 { return time.Now().UnixNano() } diff --git a/pkg/pipeline/watch.go b/pkg/pipeline/watch.go index 4ea06011..2ac1950c 100644 --- a/pkg/pipeline/watch.go +++ b/pkg/pipeline/watch.go @@ -228,15 +228,10 @@ func (c *Controller) handleMessageStateChanged(msg *gst.Message) { s := msg.Source() if s == pipelineName { - logger.Infow("pipeline playing") - - c.playing.Break() - switch c.SourceType { - case types.SourceTypeSDK: - c.updateStartTime(c.src.(*source.SDKSource).GetStartTime()) - case types.SourceTypeWeb: - c.updateStartTime(time.Now().UnixNano()) - } + c.playing.Once(func() { + logger.Infow("pipeline playing") + c.updateStartTime(c.src.GetStartedAt()) + }) } else if strings.HasPrefix(s, "app_") { s = s[4:] logger.Infow(fmt.Sprintf("%s playing", s)) diff --git a/test/integration.go b/test/integration.go index 71d008bd..20273b9d 100644 --- a/test/integration.go +++ b/test/integration.go @@ -121,14 +121,14 @@ func (r *Runner) publishSamplesToRoom(t *testing.T, audioCodec, videoCodec types return } -func (r *Runner) publishSampleOffset(t *testing.T, codec types.MimeType, publishAfter, unpublishAfter time.Duration) { +func (r *Runner) publishSampleOffset(t *testing.T, codec types.MimeType, publishAt, unpublishAt time.Duration) { if codec != "" { go func() { - time.Sleep(publishAfter) + time.Sleep(publishAt) done := make(chan struct{}) pub := r.publish(t, codec, done) - if unpublishAfter != 0 { - time.AfterFunc(unpublishAfter, func() { + if unpublishAt != 0 { + time.AfterFunc(unpublishAt-publishAt, func() { select { case <-done: return diff --git a/test/participant.go b/test/participant.go index d57bfe27..4135527a 100644 --- a/test/participant.go +++ b/test/participant.go @@ -78,8 +78,7 @@ func (r *Runner) testParticipantFile(t *testing.T) { fileType: livekit.EncodedFileType_MP4, audioCodec: types.MimeTypeOpus, videoCodec: types.MimeTypeH264, - videoDelay: time.Second * 8, - videoUnpublish: time.Second * 14, + videoUnpublish: time.Second * 10, videoRepublish: time.Second * 20, filename: "participant_{room_name}_h264_{time}.mp4", }, From d2c8c9463bb6f75d9d6a48823bbaeead6c9ca1c2 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 6 Sep 2023 18:21:56 -0700 Subject: [PATCH 18/20] add config to audio bin --- pkg/pipeline/builder/audio.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index 19cf1dd6..079f26b9 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -40,6 +40,7 @@ type AudioBin struct { func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error { b := &AudioBin{ bin: pipeline.NewBin("audio"), + p: p, tracks: make(map[string]struct{}), } From 8b85d3c4e649e2a95a7683526efcd4e22091e946 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Wed, 6 Sep 2023 23:01:07 -0700 Subject: [PATCH 19/20] use idle probe --- pkg/gstreamer/bin.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/gstreamer/bin.go b/pkg/gstreamer/bin.go index 32cb8e6f..3d84e940 100644 --- a/pkg/gstreamer/bin.go +++ b/pkg/gstreamer/bin.go @@ -173,13 +173,11 @@ func (b *Bin) RemoveSourceBin(name string) (bool, error) { } if state != StateBuilding { - logger.Debugw(fmt.Sprintf("adding probe to %s", src.bin.GetName())) src.mu.Lock() srcGhostPad, sinkGhostPad := getGhostPads(src, b) src.mu.Unlock() - srcGhostPad.AddProbe(gst.PadProbeTypeBlocking, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { - logger.Debugw("probe running") + srcGhostPad.AddProbe(gst.PadProbeTypeIdle, func(_ *gst.Pad, _ *gst.PadProbeInfo) gst.PadProbeReturn { sinkPad := sinkGhostPad.GetTarget() b.elements[0].ReleaseRequestPad(sinkPad) From 870f8348c85ebd195096904045c193ded00e9517 Mon Sep 17 00:00:00 2001 From: David Colburn Date: Thu, 7 Sep 2023 00:01:52 -0700 Subject: [PATCH 20/20] cleaning --- pkg/pipeline/builder/audio.go | 66 ++++++++++-------- pkg/pipeline/builder/video.go | 126 ++++++++++++++++++---------------- pkg/pipeline/controller.go | 1 - 3 files changed, 104 insertions(+), 89 deletions(-) diff --git a/pkg/pipeline/builder/audio.go b/pkg/pipeline/builder/audio.go index 079f26b9..7a3901ce 100644 --- a/pkg/pipeline/builder/audio.go +++ b/pkg/pipeline/builder/audio.go @@ -30,8 +30,8 @@ import ( const audioMixerLatency = uint64(2e9) type AudioBin struct { - bin *gstreamer.Bin - p *config.PipelineConfig + bin *gstreamer.Bin + conf *config.PipelineConfig mu sync.Mutex tracks map[string]struct{} @@ -40,18 +40,18 @@ type AudioBin struct { func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error { b := &AudioBin{ bin: pipeline.NewBin("audio"), - p: p, + conf: p, tracks: make(map[string]struct{}), } switch p.SourceType { case types.SourceTypeWeb: - if err := b.buildWebInput(p); err != nil { + if err := b.buildWebInput(); err != nil { return err } case types.SourceTypeSDK: - if err := b.buildSDKInput(p); err != nil { + if err := b.buildSDKInput(); err != nil { return err } @@ -67,6 +67,14 @@ func BuildAudioBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error if err = b.bin.AddElement(tee); err != nil { return err } + } else { + queue, err := gstreamer.BuildQueue("audio_queue", p.Latency, true) + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = b.bin.AddElement(queue); err != nil { + return err + } } return pipeline.AddSourceBin(b.bin) @@ -78,7 +86,7 @@ func (b *AudioBin) onTrackAdded(ts *config.TrackSource) { } if ts.Kind == lksdk.TrackKindAudio { - if err := b.addAudioAppSrcBin(b.p, ts); err != nil { + if err := b.addAudioAppSrcBin(ts); err != nil { b.bin.OnError(err) } } @@ -101,23 +109,23 @@ func (b *AudioBin) onTrackRemoved(trackID string) { } } -func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { +func (b *AudioBin) buildWebInput() error { pulseSrc, err := gst.NewElement("pulsesrc") if err != nil { return errors.ErrGstPipelineError(err) } - if err = pulseSrc.SetProperty("device", fmt.Sprintf("%s.monitor", p.Info.EgressId)); err != nil { + if err = pulseSrc.SetProperty("device", fmt.Sprintf("%s.monitor", b.conf.Info.EgressId)); err != nil { return errors.ErrGstPipelineError(err) } if err = b.bin.AddElement(pulseSrc); err != nil { return err } - if err = addAudioConverter(b.bin, p); err != nil { + if err = addAudioConverter(b.bin, b.conf); err != nil { return err } - if p.AudioTranscoding { - if err = b.addEncoder(p); err != nil { + if b.conf.AudioTranscoding { + if err = b.addEncoder(); err != nil { return err } } @@ -125,20 +133,20 @@ func (b *AudioBin) buildWebInput(p *config.PipelineConfig) error { return nil } -func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { - if p.AudioTrack != nil { - if err := b.addAudioAppSrcBin(p, p.AudioTrack); err != nil { +func (b *AudioBin) buildSDKInput() error { + if b.conf.AudioTrack != nil { + if err := b.addAudioAppSrcBin(b.conf.AudioTrack); err != nil { return err } } - if err := b.addAudioTestSrcBin(p); err != nil { + if err := b.addAudioTestSrcBin(); err != nil { return err } - if err := b.addMixer(p); err != nil { + if err := b.addMixer(); err != nil { return err } - if p.AudioTranscoding { - if err := b.addEncoder(p); err != nil { + if b.conf.AudioTranscoding { + if err := b.addEncoder(); err != nil { return err } } @@ -146,7 +154,7 @@ func (b *AudioBin) buildSDKInput(p *config.PipelineConfig) error { return nil } -func (b *AudioBin) addAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { +func (b *AudioBin) addAudioAppSrcBin(ts *config.TrackSource) error { b.mu.Lock() defer b.mu.Unlock() @@ -191,7 +199,7 @@ func (b *AudioBin) addAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackS return errors.ErrNotSupported(string(ts.MimeType)) } - if err := addAudioConverter(appSrcBin, p); err != nil { + if err := addAudioConverter(appSrcBin, b.conf); err != nil { return err } @@ -202,7 +210,7 @@ func (b *AudioBin) addAudioAppSrcBin(p *config.PipelineConfig, ts *config.TrackS return nil } -func (b *AudioBin) addAudioTestSrcBin(p *config.PipelineConfig) error { +func (b *AudioBin) addAudioTestSrcBin() error { testSrcBin := b.bin.NewBin("audio_test_src") if err := b.bin.AddSourceBin(testSrcBin); err != nil { return err @@ -222,7 +230,7 @@ func (b *AudioBin) addAudioTestSrcBin(p *config.PipelineConfig) error { return errors.ErrGstPipelineError(err) } - audioCaps, err := newAudioCapsFilter(p) + audioCaps, err := newAudioCapsFilter(b.conf) if err != nil { return err } @@ -230,7 +238,7 @@ func (b *AudioBin) addAudioTestSrcBin(p *config.PipelineConfig) error { return testSrcBin.AddElements(audioTestSrc, audioCaps) } -func (b *AudioBin) addMixer(p *config.PipelineConfig) error { +func (b *AudioBin) addMixer() error { audioMixer, err := gst.NewElement("audiomixer") if err != nil { return errors.ErrGstPipelineError(err) @@ -239,7 +247,7 @@ func (b *AudioBin) addMixer(p *config.PipelineConfig) error { return errors.ErrGstPipelineError(err) } - mixedCaps, err := newAudioCapsFilter(p) + mixedCaps, err := newAudioCapsFilter(b.conf) if err != nil { return err } @@ -247,14 +255,14 @@ func (b *AudioBin) addMixer(p *config.PipelineConfig) error { return b.bin.AddElements(audioMixer, mixedCaps) } -func (b *AudioBin) addEncoder(p *config.PipelineConfig) error { - switch p.AudioOutCodec { +func (b *AudioBin) addEncoder() error { + switch b.conf.AudioOutCodec { case types.MimeTypeOpus: opusEnc, err := gst.NewElement("opusenc") if err != nil { return errors.ErrGstPipelineError(err) } - if err = opusEnc.SetProperty("bitrate", int(p.AudioBitrate*1000)); err != nil { + if err = opusEnc.SetProperty("bitrate", int(b.conf.AudioBitrate*1000)); err != nil { return errors.ErrGstPipelineError(err) } return b.bin.AddElement(opusEnc) @@ -264,7 +272,7 @@ func (b *AudioBin) addEncoder(p *config.PipelineConfig) error { if err != nil { return errors.ErrGstPipelineError(err) } - if err = faac.SetProperty("bitrate", int(p.AudioBitrate*1000)); err != nil { + if err = faac.SetProperty("bitrate", int(b.conf.AudioBitrate*1000)); err != nil { return errors.ErrGstPipelineError(err) } return b.bin.AddElement(faac) @@ -273,7 +281,7 @@ func (b *AudioBin) addEncoder(p *config.PipelineConfig) error { return nil default: - return errors.ErrNotSupported(string(p.AudioOutCodec)) + return errors.ErrNotSupported(string(b.conf.AudioOutCodec)) } } diff --git a/pkg/pipeline/builder/video.go b/pkg/pipeline/builder/video.go index 2a9bf036..dfd91f4d 100644 --- a/pkg/pipeline/builder/video.go +++ b/pkg/pipeline/builder/video.go @@ -35,8 +35,8 @@ import ( const videoTestSrcName = "video_test_src" type VideoBin struct { - bin *gstreamer.Bin - p *config.PipelineConfig + bin *gstreamer.Bin + conf *config.PipelineConfig lastPTS atomic.Duration nextPTS atomic.Duration @@ -50,18 +50,18 @@ type VideoBin struct { func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error { b := &VideoBin{ - bin: pipeline.NewBin("video"), - p: p, + bin: pipeline.NewBin("video"), + conf: p, } switch p.SourceType { case types.SourceTypeWeb: - if err := b.buildWebInput(p); err != nil { + if err := b.buildWebInput(); err != nil { return err } case types.SourceTypeSDK: - if err := b.buildSDKInput(p); err != nil { + if err := b.buildSDKInput(); err != nil { return err } @@ -80,6 +80,14 @@ func BuildVideoBin(pipeline *gstreamer.Pipeline, p *config.PipelineConfig) error if err = b.bin.AddElement(tee); err != nil { return err } + } else { + queue, err := gstreamer.BuildQueue("video_queue", p.Latency, true) + if err != nil { + return errors.ErrGstPipelineError(err) + } + if err = b.bin.AddElement(queue); err != nil { + return err + } } return pipeline.AddSourceBin(b.bin) @@ -91,7 +99,7 @@ func (b *VideoBin) onTrackAdded(ts *config.TrackSource) { } if ts.Kind == lksdk.TrackKindVideo { - if err := b.addAppSrcBin(b.p, ts); err != nil { + if err := b.addAppSrcBin(ts); err != nil { b.bin.OnError(err) } } @@ -150,12 +158,12 @@ func (b *VideoBin) onTrackUnmuted(trackID string, pts time.Duration) { b.nextPad = trackID } -func (b *VideoBin) buildWebInput(p *config.PipelineConfig) error { +func (b *VideoBin) buildWebInput() error { xImageSrc, err := gst.NewElement("ximagesrc") if err != nil { return errors.ErrGstPipelineError(err) } - if err = xImageSrc.SetProperty("display-name", p.Display); err != nil { + if err = xImageSrc.SetProperty("display-name", b.conf.Display); err != nil { return errors.ErrGstPipelineError(err) } if err = xImageSrc.SetProperty("use-damage", false); err != nil { @@ -165,7 +173,7 @@ func (b *VideoBin) buildWebInput(p *config.PipelineConfig) error { return errors.ErrGstPipelineError(err) } - videoQueue, err := gstreamer.BuildQueue("video_input_queue", p.Latency, true) + videoQueue, err := gstreamer.BuildQueue("video_input_queue", b.conf.Latency, true) if err != nil { return err } @@ -181,7 +189,7 @@ func (b *VideoBin) buildWebInput(p *config.PipelineConfig) error { } if err = caps.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( "video/x-raw,framerate=%d/1", - p.Framerate, + b.conf.Framerate, ), )); err != nil { return errors.ErrGstPipelineError(err) @@ -191,31 +199,31 @@ func (b *VideoBin) buildWebInput(p *config.PipelineConfig) error { return err } - if p.VideoTranscoding { - if err = b.addEncoder(p); err != nil { + if b.conf.VideoTranscoding { + if err = b.addEncoder(); err != nil { return err } } return nil } -func (b *VideoBin) buildSDKInput(p *config.PipelineConfig) error { +func (b *VideoBin) buildSDKInput() error { b.pads = make(map[string]*gst.Pad) // add selector first so pads can be created - if p.VideoTranscoding { - if err := b.addSelector(p); err != nil { + if b.conf.VideoTranscoding { + if err := b.addSelector(); err != nil { return err } } - if p.VideoTrack != nil { - if err := b.addAppSrcBin(p, p.VideoTrack); err != nil { + if b.conf.VideoTrack != nil { + if err := b.addAppSrcBin(b.conf.VideoTrack); err != nil { return err } } - if p.VideoTranscoding { + if b.conf.VideoTranscoding { b.bin.SetGetSrcPad(b.getSrcPad) b.bin.SetEOSFunc(func() bool { b.mu.Lock() @@ -229,15 +237,15 @@ func (b *VideoBin) buildSDKInput(p *config.PipelineConfig) error { return false }) - if err := b.addVideoTestSrcBin(p); err != nil { + if err := b.addVideoTestSrcBin(); err != nil { return err } - if p.VideoTrack == nil { + if b.conf.VideoTrack == nil { if err := b.setSelectorPad(videoTestSrcName); err != nil { return err } } - if err := b.addEncoder(p); err != nil { + if err := b.addEncoder(); err != nil { return err } } @@ -245,13 +253,13 @@ func (b *VideoBin) buildSDKInput(p *config.PipelineConfig) error { return nil } -func (b *VideoBin) addAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource) error { - appSrcBin, err := buildVideoAppSrcBin(b.bin, p, ts) +func (b *VideoBin) addAppSrcBin(ts *config.TrackSource) error { + appSrcBin, err := b.buildAppSrcBin(ts) if err != nil { return err } - if p.VideoTranscoding { + if b.conf.VideoTranscoding { b.createSrcPad(ts.TrackID) } @@ -259,20 +267,20 @@ func (b *VideoBin) addAppSrcBin(p *config.PipelineConfig, ts *config.TrackSource return err } - if p.VideoTranscoding { + if b.conf.VideoTranscoding { return b.setSelectorPad(ts.TrackID) } return nil } -func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts *config.TrackSource) (*gstreamer.Bin, error) { - b := videoBin.NewBin(ts.TrackID) +func (b *VideoBin) buildAppSrcBin(ts *config.TrackSource) (*gstreamer.Bin, error) { + appSrcBin := b.bin.NewBin(ts.TrackID) ts.AppSrc.Element.SetArg("format", "time") if err := ts.AppSrc.Element.SetProperty("is-live", true); err != nil { return nil, errors.ErrGstPipelineError(err) } - if err := b.AddElement(ts.AppSrc.Element); err != nil { + if err := appSrcBin.AddElement(ts.AppSrc.Element); err != nil { return nil, err } @@ -300,17 +308,17 @@ func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts * return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElements(rtpH264Depay, caps); err != nil { + if err = appSrcBin.AddElements(rtpH264Depay, caps); err != nil { return nil, err } - if p.VideoTranscoding { + if b.conf.VideoTranscoding { avDecH264, err := gst.NewElement("avdec_h264") if err != nil { return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElement(avDecH264); err != nil { + if err = appSrcBin.AddElement(avDecH264); err != nil { return nil, err } } else { @@ -319,11 +327,11 @@ func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts * return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElement(h264Parse); err != nil { + if err = appSrcBin.AddElement(h264Parse); err != nil { return nil, err } - return b, nil + return appSrcBin, nil } case types.MimeTypeVP8: @@ -338,20 +346,20 @@ func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts * if err != nil { return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElement(rtpVP8Depay); err != nil { + if err = appSrcBin.AddElement(rtpVP8Depay); err != nil { return nil, err } - if p.VideoTranscoding { + if b.conf.VideoTranscoding { vp8Dec, err := gst.NewElement("vp8dec") if err != nil { return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElement(vp8Dec); err != nil { + if err = appSrcBin.AddElement(vp8Dec); err != nil { return nil, err } } else { - return b, nil + return appSrcBin, nil } case types.MimeTypeVP9: @@ -366,16 +374,16 @@ func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts * if err != nil { return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElement(rtpVP9Depay); err != nil { + if err = appSrcBin.AddElement(rtpVP9Depay); err != nil { return nil, err } - if p.VideoTranscoding { + if b.conf.VideoTranscoding { vp9Dec, err := gst.NewElement("vp9dec") if err != nil { return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElement(vp9Dec); err != nil { + if err = appSrcBin.AddElement(vp9Dec); err != nil { return nil, err } } else { @@ -394,25 +402,25 @@ func buildVideoAppSrcBin(videoBin *gstreamer.Bin, p *config.PipelineConfig, ts * return nil, errors.ErrGstPipelineError(err) } - if err = b.AddElements(vp9Parse, vp9Caps); err != nil { + if err = appSrcBin.AddElements(vp9Parse, vp9Caps); err != nil { return nil, err } - return b, nil + return appSrcBin, nil } default: return nil, errors.ErrNotSupported(string(ts.MimeType)) } - if err := addVideoConverter(b, p); err != nil { + if err := addVideoConverter(appSrcBin, b.conf); err != nil { return nil, err } - return b, nil + return appSrcBin, nil } -func (b *VideoBin) addVideoTestSrcBin(p *config.PipelineConfig) error { +func (b *VideoBin) addVideoTestSrcBin() error { testSrcBin := b.bin.NewBin(videoTestSrcName) if err := b.bin.AddSourceBin(testSrcBin); err != nil { return err @@ -427,7 +435,7 @@ func (b *VideoBin) addVideoTestSrcBin(p *config.PipelineConfig) error { } videoTestSrc.SetArg("pattern", "black") - caps, err := newVideoCapsFilter(p, true) + caps, err := newVideoCapsFilter(b.conf, true) if err != nil { return errors.ErrGstPipelineError(err) } @@ -440,7 +448,7 @@ func (b *VideoBin) addVideoTestSrcBin(p *config.PipelineConfig) error { return nil } -func (b *VideoBin) addSelector(p *config.PipelineConfig) error { +func (b *VideoBin) addSelector() error { inputSelector, err := gst.NewElement("input-selector") if err != nil { return errors.ErrGstPipelineError(err) @@ -457,7 +465,7 @@ func (b *VideoBin) addSelector(p *config.PipelineConfig) error { return err } - caps, err := newVideoCapsFilter(p, true) + caps, err := newVideoCapsFilter(b.conf, true) if err != nil { return errors.ErrGstPipelineError(err) } @@ -470,8 +478,8 @@ func (b *VideoBin) addSelector(p *config.PipelineConfig) error { return nil } -func (b *VideoBin) addEncoder(p *config.PipelineConfig) error { - videoQueue, err := gstreamer.BuildQueue("video_encoder_queue", p.Latency, false) +func (b *VideoBin) addEncoder() error { + videoQueue, err := gstreamer.BuildQueue("video_encoder_queue", b.conf.Latency, false) if err != nil { return err } @@ -479,29 +487,29 @@ func (b *VideoBin) addEncoder(p *config.PipelineConfig) error { return err } - switch p.VideoOutCodec { + switch b.conf.VideoOutCodec { // we only encode h264, the rest are too slow case types.MimeTypeH264: x264Enc, err := gst.NewElement("x264enc") if err != nil { return errors.ErrGstPipelineError(err) } - if err = x264Enc.SetProperty("bitrate", uint(p.VideoBitrate)); err != nil { + if err = x264Enc.SetProperty("bitrate", uint(b.conf.VideoBitrate)); err != nil { return errors.ErrGstPipelineError(err) } x264Enc.SetArg("speed-preset", "veryfast") - if p.KeyFrameInterval != 0 { - if err = x264Enc.SetProperty("key-int-max", uint(p.KeyFrameInterval*float64(p.Framerate))); err != nil { + if b.conf.KeyFrameInterval != 0 { + if err = x264Enc.SetProperty("key-int-max", uint(b.conf.KeyFrameInterval*float64(b.conf.Framerate))); err != nil { return errors.ErrGstPipelineError(err) } } bufCapacity := uint(2000) // 2s - if p.GetSegmentConfig() != nil { + if b.conf.GetSegmentConfig() != nil { // avoid key frames other than at segments boundaries as splitmuxsink can become inconsistent otherwise if err = x264Enc.SetProperty("option-string", "scenecut=0"); err != nil { return errors.ErrGstPipelineError(err) } - bufCapacity = uint(time.Duration(p.GetSegmentConfig().SegmentDuration) * (time.Second / time.Millisecond)) + bufCapacity = uint(time.Duration(b.conf.GetSegmentConfig().SegmentDuration) * (time.Second / time.Millisecond)) } if err = x264Enc.SetProperty("vbv-buf-capacity", bufCapacity); err != nil { return err @@ -513,7 +521,7 @@ func (b *VideoBin) addEncoder(p *config.PipelineConfig) error { } if err = caps.SetProperty("caps", gst.NewCapsFromString(fmt.Sprintf( "video/x-h264,profile=%s", - p.VideoProfile, + b.conf.VideoProfile, ))); err != nil { return errors.ErrGstPipelineError(err) } @@ -557,7 +565,7 @@ func (b *VideoBin) addEncoder(p *config.PipelineConfig) error { fallthrough default: - return errors.ErrNotSupported(fmt.Sprintf("%s encoding", p.VideoOutCodec)) + return errors.ErrNotSupported(fmt.Sprintf("%s encoding", b.conf.VideoOutCodec)) } } diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 054e0b50..81691599 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -176,7 +176,6 @@ func (c *Controller) Run(ctx context.Context) *livekit.EgressInfo { c.Info.StartedAt = time.Now().UnixNano() defer func() { now := time.Now().UnixNano() - logger.Debugw("CLOSING") if c.SourceType == types.SourceTypeSDK { c.updateDuration(c.src.GetEndedAt())