From 682cf2d3d49359d8f8678a95ccb205e73f89f819 Mon Sep 17 00:00:00 2001 From: Mario Guggenberger Date: Sat, 27 Jan 2024 14:05:02 +0100 Subject: [PATCH] feat: improve FFmpeg seeking (#26) Add three seek improvements which increase the range of file formats that can be directly decoded through FFmpeg and thus reduce the likelihood that audio proxies are required: - report container duration when stream duration is unavailable, - consider non-zero start times, - handle cases where seeks end up too early. --- nativesrc/aurioffmpegproxy/proxy.c | 8 ++- src/Aurio.FFmpeg.UnitTest/AudioDecode.cs | 12 ++++ .../Aurio.FFmpeg.UnitTest.csproj | 2 +- .../FFmpegReaderTests.cs | 56 ++++++++++++++++ .../FFmpegSourceStreamTests.cs | 23 +++++++ .../Resources/sine440-44100-16-mono-200ms.mkv | Bin 0 -> 11083 bytes .../Resources/sine440-44100-16-mono-200ms.ts | Bin 0 -> 7896 bytes src/Aurio.FFmpeg/FFmpegReader.cs | 10 +++ src/Aurio.FFmpeg/FFmpegSourceStream.cs | 61 +++++++++++++----- 9 files changed, 154 insertions(+), 18 deletions(-) create mode 100644 src/Aurio.FFmpeg.UnitTest/FFmpegReaderTests.cs create mode 100644 src/Aurio.FFmpeg.UnitTest/Resources/sine440-44100-16-mono-200ms.mkv create mode 100644 src/Aurio.FFmpeg.UnitTest/Resources/sine440-44100-16-mono-200ms.ts diff --git a/nativesrc/aurioffmpegproxy/proxy.c b/nativesrc/aurioffmpegproxy/proxy.c index 9409e9f1..6b9dcbf8 100644 --- a/nativesrc/aurioffmpegproxy/proxy.c +++ b/nativesrc/aurioffmpegproxy/proxy.c @@ -187,8 +187,12 @@ ProxyInstance *stream_open(ProxyInstance *pi) pi->audio_output.format.channels); } - pi->audio_output.length = pi->audio_stream->duration != AV_NOPTS_VALUE ? - pts_to_samples(pi->audio_output.format.sample_rate, pi->audio_stream->time_base, pi->audio_stream->duration) : AV_NOPTS_VALUE; + pi->audio_output.length = + pi->audio_stream->duration != AV_NOPTS_VALUE + ? pts_to_samples(pi->audio_output.format.sample_rate, pi->audio_stream->time_base, pi->audio_stream->duration) + : pi->fmt_ctx->duration != AV_NOPTS_VALUE + ? pts_to_samples(pi->audio_output.format.sample_rate, AV_TIME_BASE_Q, pi->fmt_ctx->duration) + : AV_NOPTS_VALUE; /* * TODO To get the frame size, read the first frame, take the size, and seek back to the start. diff --git a/src/Aurio.FFmpeg.UnitTest/AudioDecode.cs b/src/Aurio.FFmpeg.UnitTest/AudioDecode.cs index 307bf9c0..2e5c95d5 100644 --- a/src/Aurio.FFmpeg.UnitTest/AudioDecode.cs +++ b/src/Aurio.FFmpeg.UnitTest/AudioDecode.cs @@ -26,5 +26,17 @@ public void Mp3ReadData() var length = StreamUtil.ReadAllAndCount(s); Assert.True(length > 46000); } + + [Fact] + public void TS_ReadDataUntilEnd() + { + var s = new FFmpegSourceStream( + new FileInfo("./Resources/sine440-44100-16-mono-200ms.mp3") + ); + + StreamUtil.ReadAllAndCount(s); + + // Test succeeds when reading does not get stuck in infinite loop at EOF + } } } diff --git a/src/Aurio.FFmpeg.UnitTest/Aurio.FFmpeg.UnitTest.csproj b/src/Aurio.FFmpeg.UnitTest/Aurio.FFmpeg.UnitTest.csproj index ced1c761..cc61d17b 100644 --- a/src/Aurio.FFmpeg.UnitTest/Aurio.FFmpeg.UnitTest.csproj +++ b/src/Aurio.FFmpeg.UnitTest/Aurio.FFmpeg.UnitTest.csproj @@ -26,7 +26,7 @@ - + PreserveNewest diff --git a/src/Aurio.FFmpeg.UnitTest/FFmpegReaderTests.cs b/src/Aurio.FFmpeg.UnitTest/FFmpegReaderTests.cs new file mode 100644 index 00000000..c8c7b52a --- /dev/null +++ b/src/Aurio.FFmpeg.UnitTest/FFmpegReaderTests.cs @@ -0,0 +1,56 @@ +using System; +using System.IO; +using Xunit; + +namespace Aurio.FFmpeg.UnitTest +{ + public class FFmpegReaderTests + { + /// + /// MKV does not carry individual stream lenghts, only a total container length. + /// + [Fact] + public void MKV_LengthFromContainerAvailable() + { + var fileInfo = new FileInfo("./Resources/sine440-44100-16-mono-200ms.mkv"); + + var reader = new FFmpegReader(fileInfo, Type.Audio); + + Assert.NotEqual(long.MinValue, reader.AudioOutputConfig.length); + } + + [Fact] + public void TS_NonZeroStartTime() + { + var fileInfo = new FileInfo("./Resources/sine440-44100-16-mono-200ms.ts"); + var reader = new FFmpegReader(fileInfo, Type.Audio); + var sourceBuffer = new byte[reader.FrameBufferSize]; + var startTimeSecs = 2000; + + reader.ReadFrame(out long readerPosition, sourceBuffer, sourceBuffer.Length, out _); + + Assert.Equal( + startTimeSecs * reader.AudioOutputConfig.format.sample_rate, + readerPosition + ); + } + + [Fact] + public void SignalEOF() + { + var fileInfo = new FileInfo("./Resources/sine440-44100-16-mono-200ms.ts"); + var reader = new FFmpegReader(fileInfo, Type.Audio); + var sourceBuffer = new byte[reader.FrameBufferSize]; + int result; + + // Read over all frames (seeking does not work here due to short stream) + do + { + result = reader.ReadFrame(out _, sourceBuffer, sourceBuffer.Length, out _); + } while (result > 0); + + // -1 signals EOF + Assert.Equal(-1, result); + } + } +} diff --git a/src/Aurio.FFmpeg.UnitTest/FFmpegSourceStreamTests.cs b/src/Aurio.FFmpeg.UnitTest/FFmpegSourceStreamTests.cs index c0bee4d3..b60ecc32 100644 --- a/src/Aurio.FFmpeg.UnitTest/FFmpegSourceStreamTests.cs +++ b/src/Aurio.FFmpeg.UnitTest/FFmpegSourceStreamTests.cs @@ -78,5 +78,28 @@ public void SuggestWaveProxyFileInfo_WithDirectory_Unix() proxyFileInfo.FullName ); } + + [Fact] + public void MKV_NoStreamDuration_SeekingSupported() + { + var fileInfo = new FileInfo("./Resources/sine440-44100-16-mono-200ms.mkv"); + + var act = () => new FFmpegSourceStream(fileInfo); + var ex = Record.Exception(act); + + // Assert no FileNotSeekableException being thrown + Assert.Null(ex); + } + + [Fact] + public void TS_NonZeroStartTime_SeekingSupported() + { + var fileInfo = new FileInfo("./Resources/sine440-44100-16-mono-200ms.ts"); + var s = new FFmpegSourceStream(fileInfo); + + s.Position = 1000; + + Assert.Equal(1000, s.Position); + } } } diff --git a/src/Aurio.FFmpeg.UnitTest/Resources/sine440-44100-16-mono-200ms.mkv b/src/Aurio.FFmpeg.UnitTest/Resources/sine440-44100-16-mono-200ms.mkv new file mode 100644 index 0000000000000000000000000000000000000000..3137d9c898889fcbaac54324ca4237abd832214f GIT binary patch literal 11083 zcmeI22~-p37Jw%KltuP}5v_!<6wxG;kU&U@Bq2ed1j15rp_<8L0wZKG37gNJmmmlf z5sFq6wMD52qPA39+qw|MT3_oft|+2NTU!^XwJVx835&MAzE^yBCd4nsx{3^VLb5&`2biy^<6T8- zb;g#$?0;U~CfZ$=+*Ifh(qJie5!{G6V7D zTL!=!AMxduLZ4US?X_BQEo{`MAF34A70nRFCkX|`Mf|L6M4Ns{j0)?P#|VOF(&vkl zVrEiiiNu0paY}eZo*IHP5gk#Aq@zj>DJ>6?r{zhd*hfwxqB0ICl_>&6>PRF5jb5Zd zk{89Nz>6~BFc~RgdRiU`h3B#J6lw*cgNS*GETxv6$00#7l@wuPf1A6hB%)=CE|Wvz znFolNZDOQKgAkdN5Hbvggc3u;C^TkhC?j+lkq*+BR4|wt43g==YzD|?(20W=$zqTg zu93;vNrI?BdsvG@%FyZ5Y$`Q3H;@JNLUy^$ZU1cC52ib8aATTQY;QqV3mT3`A(%#K&l{L z2`Nxmgve0L<444QM;ICqessR*_sk7SC=|anqSXm=u$2!6;@4{&6w;>Ts}X7vqE%&U zU__XMD0RNSF)f3KRj9KyS(Zr3U@DS@D6rXD%vIX3Va#>52urXG(m?`9hj2)AkRD8? z(a1Db3XQ>LGTBVlRFK97!67Ra5tR&;=l^j5k%%3m)aoE5jKl~yBuoVbMP+PS7|a6Y zFa{8Wz+?u*kdmb=dMFu|v*-+%0f$IKWDl9evB zV{QsW>=X^QIkf}WV>DXG=R@hk9RodC=n!vaV9jzmvs^r+z;Y6qjfhpKQb!a)noLBa z9rDHk_wZaZ8J4+^49hi`nj}klxp?sM08wK1aJd6-1i_^b?R&7X41VzHBmK|@ zVO@#fJMx32d$A1}DweWm}^Q=O9*M=U=X;}&aEWMw!JxH6VJCJKs1xvd9r zMqA0PO=ECeBD4{XN@$fqEW7$wcnrxHU}>@b+{=L4s0BR%yxv{!HNHS>4_{CHk!H_3 zlj7bpzke(~ns&hW59dv@`%K;XsjcGpu@m}_?s>Dq)Gu=4=kQ#|)JX1Y=OxTAnfCV@ zCti(~yS*oF|73F1ls=Q`&bhZ%w4J*g>eRj0)qXMma#4%PWI7SU;PtOb^ZwTx*S`It z<7ntQ73(tA-T%FRlzsHBiCd%>m7B{Jezu*lJndGGW^edT>t+BT=>XuPM7mp6FZNoC zf-aaD*Z@dAT&dcdw7|y;IzF9%Z%ovb#m#AWf3y6k?S8YJzw5Gq=Vbve8KBLYnsDia zp18RP4ZGB}^6y&HUupLqmm8zf0|0Jg2@tSqUZk}!I5If7EezD-JGSV=msHA_8uN--`ghqmCJv6ZCSu$ zSisv1L|pTI>1f}z#vGfc6TAjnSF(ApGkLc7lMY8Em9-g1x%o>Ey5S}IXx3{;!&tYm z&CZ`s>DH|odB%R^xiID)$F%cfvfc=M4gfPEnA000_>~4J(UUO_2fEhU#7+#HZ{;v@ z-Or$e1-$qn@Oph?j^2EF!Mp2<_uq%1F=n|Z;IW5bMFcp6h;|k8t3qO)^oEP7}hX)O2iE7pSy%b_7z71NM^2Ub}=N4M*y6EhAU{>`7R_ zTQUURSJ+W|$GS162+i-5JbmDui;nAHVHI!UY7n1xg$uOyrJX4c@9I8f2Q2d(OPPbi z5!Lj|Wy)*Hs;W-!5hnY7;vjewu?sG(m22I4V%8bQj4rqqM69Yy_1q|1CqvD>+10bN zUKpY3XK}g(y!F2UFZ(G6&x`AagNm9!eG>B+gQf4lxVVl-`WM7yLT2`nAUf7UiH~CKfH8`wf*ikQ`1aru==cN z&ZXdG9dkQde(v1GvAV#OZgh{*1_0|0Nh&9I%*`tQ5bbV!=QAUO78?&rNo!wtE@vlH zHMZUg@7c(i>smM#Nt7&$LSI?hQl;=*+DuU1lHb^=Y>;=QDe3|a+xjnkK4SqlU=<;Q zU!TL0TL&aX!w0^g#6gAigm}9ESGQHRMKks<;U;PUH!v~S$^op6bi~;>jd3n>a&=rW zZl!~r!x+Y-+C=XDX$=e}TKh?fi*zY?>p!g>vWTo73OfX@%o~ajrO{%SF)CR@pz#x~TPdL)@IHp2Zu^ zEtpU$fKQzFtBP}=z&}3%$-mkizE`h%Zc%>`pjQba#1>engzTQ zgW$1y>JqlA9N!ziwmy3qM_GOB#6KS&z9%SI5EJwgTu$OrHD%RM!0Ao%eXpwvu}24Y z_pNyj<$Ii1eYS;qKc4uL&h_KUCloq(R30w)l1tj1=~!MKRI?PcnOnY3cX+*DfQCHl z;KfqP0qe%3h)vg4Md#LBzIVo5_bzqW{(|t-FU`aJS$&yX9Q z9@0;Q&ZhC-JVA#>RhcZ{eKT~__TnhmEs{?T;BN!~>(TK1Dgsz!Mz1Fp0rS^atFixM ZFj_ literal 0 HcmV?d00001 diff --git a/src/Aurio.FFmpeg.UnitTest/Resources/sine440-44100-16-mono-200ms.ts b/src/Aurio.FFmpeg.UnitTest/Resources/sine440-44100-16-mono-200ms.ts new file mode 100644 index 0000000000000000000000000000000000000000..b99a12d1b237bbe5c5325b5f8575f2c61e3b8d8d GIT binary patch literal 7896 zcmds*c~}!?9>?EF0)!)wLqa&@3PeD`fT6|;l7NImECH-2AcRW^mk5f-(%q4A79)t% zC>}*!j(}28cC8jAfTFdCpol1fDBu?Bk*r5;H=2EsZd-iRrzk4!`^P-8?vE9ypr)9T5S){EzeKGhLl0Z94#Q)_37a@S zM8(^t8@qOPO53u96aP!3hE&Ue(Fjg)1cLk7c630NH| z(%e9Km*(0_vrlb{+}I`D?STLiY!u783691?KK1!X=((c-)BB}?hwc`&upb20_c(4{ zA$#(4XkXKDlT)y=y7Ac4p{n})?0Y}WU!q8J$v+xaJG8c5-v-i3bL}%u1`UdG=)h87?gq< z=%w>DB8LJW+iP$;7f=Sn6lxex`xDZ?=#(y&EBn!@4g62k^cAgjyYiKgovfIz?V8FZ zsTK>KriA)>D3y&eso@RJrR0PBj+4ebYq?T+zw4{bO7r z7285gzWZ|E-#5EkXYN?ez6P5Wk1Tl_o`(%9Bd#Wv=dCzX&Dt1pcQE;wQ!TCr0P~dq z9Px7?$IIYzwiCuKw5kAtZ;F;5^N%$%2ydH1!$|y+^L&CT{!zZP72h~K&dUm9r@X9h z+}fe~0)YLZslGk!sA+koa8^%W(KsZB?32bJ&D%}!gv0S#j>6Y*PYz&g33bD{&v2U& z&;;3aXE9MscjZdH?IPE{c}WAu3+;VTt{E$?4?b0&C`|Q@zOCLChPtbbGD3)!JVs?m zpXmaip{pD62=DuLpMI8>1N3>V@&O5C*Gp9yF9p8a`-JSY3Ovbn z_^o&3;xXqnTLlFrF|C}nSl5nZ81R;$tZ-nxI~wz0!K<`P-KZ7{p5SNxY& zlK?Zu#c__r#kF94gejG!g)m7aD&B@xVOY&E=Z5V1V-A`i0g&C`jZj7uD5HC|-Wia* zTlGP9!#83d75tEdu{;j-p}LU@bn9qXnhm4aD|?qjk_l*52FQ6+1B|wldZLl|JkMG- zD)P@%ep8=c`~o}?*$UZ>9Lx-I!%c*IPNg2Rz!O=og_^lm?par~?9U{8kJmtUXM6A- z0p4I}=-){nuzx3g!2W&Cuzx?iYqWn)Sv&0CpRHjdUyxx3RW}|E>J3ElTqxAL$#YFF ztXAacKU=?My7p8FEL50_yjjN|J2#mUFXyOGx#kTh(4yb4x@=O_8}`EZS2ilRX6_^I z$zN&bf05>7UEIBUhgG764gmzV0S!TcOx^cv#o02X{*QSeJMoY859u7TlioV-i28nj z4(dKmlA0-8+bb9k^+9&>TW23NK$oXaHyf^BhFh7Kkn0@KAP?yjIra5lPzM{RYIZ>g zVw&V|EiZ8(v9Pe$WK`&L70CXgebZCFUlnM_Cn6yY99%>DqE13gUt|e08Y|94IXt}f z`r;>x#f#&oJeTp^RU!Mt-#7g}9i!9-*>zWqn*x6yR!ZWLwsK4Vm z78~trESaH#XbR9I0S%XZw@FDWFR%G(tWeOuyZ`Fnmld5bz|gELiv!Yk)#R|*Ocf!G zIV*b;R)o2doxmiFH8o_{8*9u&xgN69-iYOy0_}~Uf4_&T@|`L?*5mxo*H8E~!hEiX zWTLDstTyEnOZou5pJiKo+3#_4Erk}{KxG>+?QocRb z`QiQ8E7RmRLdBOlf0}oJUHx#svT7mh6kk#X^~~7N9o$?0OYdQ4>=jPL9ttmo0d|}g z6c~30$BVuZ_DjA#BMlcqc7tqbbA{5FC<|vqXlo7H901h!`MQLaV<+_FU2wZJtI%twDSqiD5HI%&LEMu z;4?iMKj-+R05a2W23C_<;zmGJv{iNTv`K``T1A@bnyM_L5EqGz`Mll zs1b=^NOGo7NG9iN>@L_lby)h@bo}f;=*W-uF7v)pEB-64J=@DYIQv&Zh=&vZrH zi@RO3`4F${Vll;Wd2xzuT+85owjE_f;@QhJbvlM6cfQpQE>L`YkKX&U^SNmROugOx zU+&ELx)zIYnLZ>$O1#QcHWesBahVNm6+S^zsoA@`V)b+9Mz&wJD0Cs_Hq%$+rukdb zH+u~@SnUf6))#q4`_UexdYKfSu}sdn8WA=P`gigN?BB^Buzz*M*KVO_Q?PD zI{Aoy|9Eh(Z}oD*LA~u|t2Q_%iqEzG?e)JR;~1IlWit=zeR8D8oHIQ+PaMv;SQKm(D#4=TD_eG2 zFp_N-HDArOJAsquN8q+&2YKxm!6vx5+4EOWAes)XY_ineS zAKj+vMx1olQR@BpdN&2xjbAVF{YOD|lebPW;-F-R9kjk?slZYS0B#%xh3x-#`zAO; T6TZNCcOZ=Vw>06Ae*^q4Rsax; literal 0 HcmV?d00001 diff --git a/src/Aurio.FFmpeg/FFmpegReader.cs b/src/Aurio.FFmpeg/FFmpegReader.cs index e55dcf70..c3ef8464 100644 --- a/src/Aurio.FFmpeg/FFmpegReader.cs +++ b/src/Aurio.FFmpeg/FFmpegReader.cs @@ -204,6 +204,16 @@ public VideoOutputConfig VideoOutputConfig get { return videoOutputConfig; } } + public long FrameBufferSize + { + get + { + return AudioOutputConfig.frame_size + * AudioOutputConfig.format.channels + * AudioOutputConfig.format.sample_size; + } + } + public int ReadFrame( out long timestamp, byte[] output_buffer, diff --git a/src/Aurio.FFmpeg/FFmpegSourceStream.cs b/src/Aurio.FFmpeg/FFmpegSourceStream.cs index 36b98865..7bdd17e1 100644 --- a/src/Aurio.FFmpeg/FFmpegSourceStream.cs +++ b/src/Aurio.FFmpeg/FFmpegSourceStream.cs @@ -91,11 +91,7 @@ public FFmpegSourceStream(Stream stream, string fileName) ); readerPosition = 0; - sourceBuffer = new byte[ - reader.AudioOutputConfig.frame_size - * reader.AudioOutputConfig.format.channels - * reader.AudioOutputConfig.format.sample_size - ]; + sourceBuffer = new byte[reader.FrameBufferSize]; sourceBufferPosition = 0; sourceBufferLength = -1; // -1 means buffer empty, >= 0 means valid buffer data @@ -133,7 +129,50 @@ public long Length private long SamplePosition { - get { return readerPosition + sourceBufferPosition; } + get { return readerPosition + sourceBufferPosition - readerFirstPTS; } + } + + /// + /// Read frames (repeatedly) into the buffer until it contains the sample with the + /// desired timestamp. + /// + /// This is a helper method for sample-exact seeking, because FFmpeg seeks may end + /// up a long way before the desired seek target. + /// + private void ForwardReadUntilTimestamp(long targetTimestamp) + { + long previousReaderPosition = long.MinValue; + + while (true) + { + sourceBufferLength = reader.ReadFrame( + out readerPosition, + sourceBuffer, + sourceBuffer.Length, + out Type type + ); + + if (readerPosition == previousReaderPosition) + { + // Prevent an endless read-loop in case the reported position does not change. + // I did not encounter this behavior, but who knows how FFmpeg acts on the myriad of supported formats. + throw new InvalidOperationException("Read head is stuck"); + } + else if (targetTimestamp < readerPosition) + { + // Prevent endless loop in case the target timestamp gets skipped. Again, I + // have not seen it happen, so this is just another proactive measure. + throw new InvalidOperationException( + "Read position is beyond the target timestamp" + ); + } + else if (targetTimestamp < readerPosition + sourceBufferLength) + { + break; + } + + previousReaderPosition = readerPosition; + } } public long Position @@ -145,17 +184,9 @@ public long Position // seek to target position reader.Seek(seekTarget, FFmpeg.Type.Audio); - - // get target position - sourceBufferLength = reader.ReadFrame( - out readerPosition, - sourceBuffer, - sourceBuffer.Length, - out Type type - ); + ForwardReadUntilTimestamp(seekTarget); // check if seek ended up at seek target (or earlier because of frame size, depends on file format and stream codec) - // TODO handle seek offset with bufferPosition if (seekTarget == readerPosition) { // perfect case