From 5e1638c42fbe2e157838096e380691deb5954479 Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 13:26:56 +0200 Subject: [PATCH 1/8] Rewrite resampler --- resample.go | 123 +++++++++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 55 deletions(-) diff --git a/resample.go b/resample.go index 28d494f..d3315bd 100644 --- a/resample.go +++ b/resample.go @@ -5,6 +5,8 @@ import ( "math" ) +const resamplerSingleBufferSize = 512 + // Resample takes a Streamer which is assumed to stream at the old sample rate and returns a // Streamer, which streams the data from the original Streamer resampled to the new sample rate. // @@ -53,12 +55,12 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { return &Resampler{ s: s, ratio: ratio, - first: true, - buf1: make([][2]float64, 512), - buf2: make([][2]float64, 512), + buf1: make([][2]float64, resamplerSingleBufferSize), + buf2: make([][2]float64, resamplerSingleBufferSize), pts: make([]point, quality*2), - off: 0, + off: -resamplerSingleBufferSize, pos: 0, + end: math.MaxInt, } } @@ -68,77 +70,88 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { type Resampler struct { s Streamer // the orignal streamer ratio float64 // old sample rate / new sample rate - first bool // true when Stream was not called before buf1, buf2 [][2]float64 // buf1 contains previous buf2, new data goes into buf2, buf1 is because interpolation might require old samples pts []point // pts is for points used for interpolation off int // off is the position of the start of buf2 in the original data pos int // pos is the current position in the resampled data + end int // end is the position after the last sample in the original data } // Stream streams the original audio resampled according to the current ratio. func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { - // if it's the first time, we need to fill buf2 with initial data, buf1 remains zeroed - if r.first { - sn, _ := r.s.Stream(r.buf2) - r.buf2 = r.buf2[:sn] - r.first = false - } - // we start resampling, sample by sample for len(samples) > 0 { - again: - for c := range samples[0] { - // calculate the current position in the original data - j := float64(r.pos) * r.ratio + // Calculate the current position in the original data. + wantPos := float64(r.pos) * r.ratio - // find quality*2 closest samples to j and translate them to points for interpolation - for pi := range r.pts { - // calculate the index of one of the closest samples - k := int(j) + pi - len(r.pts)/2 + 1 + // Determine the quality*2 closest sample positions for the interpolation. + // The window has length len(r.pts) and is centered around wantPos. + // todo: check if this is actually centered around wantPos + windowStart := int(wantPos) - len(r.pts)/2 + 1 // (inclusive) + windowEnd := int(wantPos) + len(r.pts)/2 + 1 // (exclusive) - var y float64 - switch { - // the sample is in buf1 - case k < r.off: - y = r.buf1[len(r.buf1)+k-r.off][c] - // the sample is in buf2 - case k < r.off+len(r.buf2): - y = r.buf2[k-r.off][c] - // the sample is beyond buf2, so we need to load new data - case k >= r.off+len(r.buf2): - // we load into buf1 - sn, _ := r.s.Stream(r.buf1) - // this condition happens when the original Streamer got - // drained and j is after the end of the - // original data - if int(j) >= r.off+len(r.buf2)+sn { - return n, n > 0 - } - // this condition happens when the original Streamer got - // drained and this one of the closest samples is after the - // end of the original data - if k >= r.off+len(r.buf2)+sn { - y = 0 - break - } - // otherwise everything is fine, we swap buffers and start - // calculating the sample again - r.off += len(r.buf2) - r.buf1 = r.buf1[:sn] - r.buf1, r.buf2 = r.buf2, r.buf1 - goto again + // Prepare the buffers. + if windowEnd >= r.off+resamplerSingleBufferSize { + // We load into buf1. + sn, _ := r.s.Stream(r.buf1) + if sn < len(r.buf1) { + r.end = r.off + resamplerSingleBufferSize + sn + + // Zero the rest of the buffer + for i := sn; i < len(r.buf1); i++ { + r.buf1[i] = [2]float64{} } + } + + // Swap buffers. + r.buf1, r.buf2 = r.buf2, r.buf1 + r.off += resamplerSingleBufferSize + } - r.pts[pi] = point{float64(k), y} + // Exit when wantPos is after the end of the original data. + if int(wantPos) >= r.end { + return n, n > 0 + } + + // Adjust the window to be within the available buffers. + //if windowStart < 0 { + // windowStart = 0 + //} + //if windowEnd > r.end { + // windowEnd = r.end + //} + + // For each channel... + for c := range samples[0] { + // Get the points. + numPts := windowEnd - windowStart + pts := r.pts[:numPts] + for i := range pts { + x := windowStart + i + var y float64 + if x < r.off { + // Sample is in buf1. + offBuf1 := r.off - resamplerSingleBufferSize + y = r.buf1[x-offBuf1][c] + } else { + // Sample is in buf2. + y = r.buf2[x-r.off][c] + } + pts[i] = point{ + X: float64(x), + Y: y, + } } - // calculate the resampled sample using polynomial interpolation from the - // quality*2 closest samples - samples[0][c] = lagrange(r.pts, j) + // Calculate the resampled sample using polynomial interpolation from the + // quality*2 closest samples. + samples[0][c] = lagrange(r.pts, wantPos) } + samples = samples[1:] n++ r.pos++ } + return n, true } From 7301df404c6317209bd59e261cb44c3e9fd9e898 Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 14:07:31 +0200 Subject: [PATCH 2/8] Make the resampler window symmetric & center it --- resample.go | 7 +++---- resample_test.go | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/resample.go b/resample.go index d3315bd..d30971c 100644 --- a/resample.go +++ b/resample.go @@ -57,7 +57,7 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { ratio: ratio, buf1: make([][2]float64, resamplerSingleBufferSize), buf2: make([][2]float64, resamplerSingleBufferSize), - pts: make([]point, quality*2), + pts: make([]point, quality*2+1), off: -resamplerSingleBufferSize, pos: 0, end: math.MaxInt, @@ -85,9 +85,8 @@ func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { // Determine the quality*2 closest sample positions for the interpolation. // The window has length len(r.pts) and is centered around wantPos. - // todo: check if this is actually centered around wantPos - windowStart := int(wantPos) - len(r.pts)/2 + 1 // (inclusive) - windowEnd := int(wantPos) + len(r.pts)/2 + 1 // (exclusive) + windowStart := int(wantPos) - len(r.pts)/2 // (inclusive) + windowEnd := int(wantPos) + len(r.pts)/2 + 1 // (exclusive) // Prepare the buffers. if windowEnd >= r.off+resamplerSingleBufferSize { diff --git a/resample_test.go b/resample_test.go index 2d5ed3d..acce006 100644 --- a/resample_test.go +++ b/resample_test.go @@ -32,7 +32,7 @@ func TestResample(t *testing.T) { func resampleCorrect(quality int, old, new beep.SampleRate, p [][2]float64) [][2]float64 { ratio := float64(old) / float64(new) - pts := make([]point, quality*2) + pts := make([]point, quality*2+1) var resampled [][2]float64 for i := 0; ; i++ { j := float64(i) * ratio @@ -42,7 +42,7 @@ func resampleCorrect(quality int, old, new beep.SampleRate, p [][2]float64) [][2 var sample [2]float64 for c := range sample { for k := range pts { - l := int(j) + k - len(pts)/2 + 1 + l := int(j) + k - (len(pts)-1)/2 if l >= 0 && l < len(p) { pts[k] = point{X: float64(l), Y: p[l][c]} } else { From 3d1c089f957392c6cfb76f2702353a26af4e9b5d Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 15:17:08 +0200 Subject: [PATCH 3/8] Only use samples within the streamed data for resampling --- resample.go | 19 +++++++------------ resample_test.go | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/resample.go b/resample.go index d30971c..99e6c8a 100644 --- a/resample.go +++ b/resample.go @@ -94,11 +94,6 @@ func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { sn, _ := r.s.Stream(r.buf1) if sn < len(r.buf1) { r.end = r.off + resamplerSingleBufferSize + sn - - // Zero the rest of the buffer - for i := sn; i < len(r.buf1); i++ { - r.buf1[i] = [2]float64{} - } } // Swap buffers. @@ -112,12 +107,12 @@ func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { } // Adjust the window to be within the available buffers. - //if windowStart < 0 { - // windowStart = 0 - //} - //if windowEnd > r.end { - // windowEnd = r.end - //} + if windowStart < 0 { + windowStart = 0 + } + if windowEnd > r.end { + windowEnd = r.end + } // For each channel... for c := range samples[0] { @@ -143,7 +138,7 @@ func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { // Calculate the resampled sample using polynomial interpolation from the // quality*2 closest samples. - samples[0][c] = lagrange(r.pts, wantPos) + samples[0][c] = lagrange(pts, wantPos) } samples = samples[1:] diff --git a/resample_test.go b/resample_test.go index acce006..0539cb6 100644 --- a/resample_test.go +++ b/resample_test.go @@ -49,7 +49,22 @@ func resampleCorrect(quality int, old, new beep.SampleRate, p [][2]float64) [][2 pts[k] = point{X: float64(l), Y: 0} } } - y := lagrange(pts[:], j) + + startK := 0 + for k, pt := range pts { + if pt.X >= 0 { + startK = k + break + } + } + endK := 0 + for k, pt := range pts { + if pt.X < float64(len(p)) { + endK = k + 1 + } + } + + y := lagrange(pts[startK:endK], j) sample[c] = y } resampled = append(resampled, sample) From 79137fc13d8d796061cf8cc937ff405a6fbb3377 Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 15:45:20 +0200 Subject: [PATCH 4/8] Revert resampler pts length increase The original length of pts made sense because when wantPos is a decimal position like 3.5, you want the window for quality 1 to contain positions 3 and 4. Having a window of 1 more doesn't make sense for that. My previous reasoning for increasing the window size was arguing from the point of wantPosition always being an integer value, which isn't the case. --- resample.go | 6 +++--- resample_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/resample.go b/resample.go index 99e6c8a..e79e2a7 100644 --- a/resample.go +++ b/resample.go @@ -57,7 +57,7 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { ratio: ratio, buf1: make([][2]float64, resamplerSingleBufferSize), buf2: make([][2]float64, resamplerSingleBufferSize), - pts: make([]point, quality*2+1), + pts: make([]point, quality*2), off: -resamplerSingleBufferSize, pos: 0, end: math.MaxInt, @@ -85,8 +85,8 @@ func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { // Determine the quality*2 closest sample positions for the interpolation. // The window has length len(r.pts) and is centered around wantPos. - windowStart := int(wantPos) - len(r.pts)/2 // (inclusive) - windowEnd := int(wantPos) + len(r.pts)/2 + 1 // (exclusive) + windowStart := int(wantPos) - (len(r.pts)-1)/2 // (inclusive) + windowEnd := int(wantPos) + len(r.pts)/2 + 1 // (exclusive) // Prepare the buffers. if windowEnd >= r.off+resamplerSingleBufferSize { diff --git a/resample_test.go b/resample_test.go index 0539cb6..d3009e9 100644 --- a/resample_test.go +++ b/resample_test.go @@ -32,7 +32,7 @@ func TestResample(t *testing.T) { func resampleCorrect(quality int, old, new beep.SampleRate, p [][2]float64) [][2]float64 { ratio := float64(old) / float64(new) - pts := make([]point, quality*2+1) + pts := make([]point, quality*2) var resampled [][2]float64 for i := 0; ; i++ { j := float64(i) * ratio From 7ea20bc9e30205b38e4970aa3ad74b3eae57ae5d Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 19:46:01 +0200 Subject: [PATCH 5/8] Fuzz & fix Resampler Two bugs: - SetRatio() could cause out of range panic when the ratio was sufficiently big. This happens when the newly calculated position was casted to an int. This caused the value to be rounded down. If difference times the ratio was bigger than the buffer, the calculated wantPos could be a negative index in buf1. - When the ratio is sufficiently big, the Stream function may not only need to fill the buffers with new data once, but multiple times. The if statement has been replaced with a for loop, so the check is executed until the desired samples are reached. --- resample.go | 14 +++++++------- resample_test.go | 34 ++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/resample.go b/resample.go index e79e2a7..7958e99 100644 --- a/resample.go +++ b/resample.go @@ -49,7 +49,7 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { if quality < 1 || 64 < quality { panic(fmt.Errorf("resample: invalid quality: %d", quality)) } - if math.IsInf(ratio, 0) || math.IsNaN(ratio) { + if ratio <= 0 || math.IsInf(ratio, 0) || math.IsNaN(ratio) { panic(fmt.Errorf("resample: invalid ratio: %f", ratio)) } return &Resampler{ @@ -59,7 +59,7 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { buf2: make([][2]float64, resamplerSingleBufferSize), pts: make([]point, quality*2), off: -resamplerSingleBufferSize, - pos: 0, + pos: 0.0, end: math.MaxInt, } } @@ -73,7 +73,7 @@ type Resampler struct { buf1, buf2 [][2]float64 // buf1 contains previous buf2, new data goes into buf2, buf1 is because interpolation might require old samples pts []point // pts is for points used for interpolation off int // off is the position of the start of buf2 in the original data - pos int // pos is the current position in the resampled data + pos float64 // pos is the current position in the resampled data end int // end is the position after the last sample in the original data } @@ -81,7 +81,7 @@ type Resampler struct { func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { for len(samples) > 0 { // Calculate the current position in the original data. - wantPos := float64(r.pos) * r.ratio + wantPos := r.pos * r.ratio // Determine the quality*2 closest sample positions for the interpolation. // The window has length len(r.pts) and is centered around wantPos. @@ -89,7 +89,7 @@ func (r *Resampler) Stream(samples [][2]float64) (n int, ok bool) { windowEnd := int(wantPos) + len(r.pts)/2 + 1 // (exclusive) // Prepare the buffers. - if windowEnd >= r.off+resamplerSingleBufferSize { + for windowEnd > r.off+resamplerSingleBufferSize { // We load into buf1. sn, _ := r.s.Stream(r.buf1) if sn < len(r.buf1) { @@ -161,10 +161,10 @@ func (r *Resampler) Ratio() float64 { // SetRatio sets the resampling ratio. This does not cause any glitches in the stream. func (r *Resampler) SetRatio(ratio float64) { - if math.IsInf(ratio, 0) || math.IsNaN(ratio) { + if ratio <= 0 || math.IsInf(ratio, 0) || math.IsNaN(ratio) { panic(fmt.Errorf("resample: invalid ratio: %f", ratio)) } - r.pos = int(float64(r.pos) * r.ratio / ratio) + r.pos *= r.ratio / ratio r.ratio = ratio } diff --git a/resample_test.go b/resample_test.go index d3009e9..e9724c3 100644 --- a/resample_test.go +++ b/resample_test.go @@ -1,6 +1,7 @@ package beep_test import ( + "fmt" "reflect" "testing" @@ -16,15 +17,17 @@ func TestResample(t *testing.T) { continue // skip too expensive combinations } - s, data := testtools.RandomDataStreamer(numSamples) + t.Run(fmt.Sprintf("numSamples_%d_old_%d_new_%d", numSamples, old, new), func(t *testing.T) { + s, data := testtools.RandomDataStreamer(numSamples) - want := resampleCorrect(3, old, new, data) + want := resampleCorrect(3, old, new, data) - got := testtools.Collect(beep.Resample(3, old, new, s)) + got := testtools.Collect(beep.Resample(3, old, new, s)) - if !reflect.DeepEqual(want, got) { - t.Fatal("Resample not working correctly") - } + if !reflect.DeepEqual(want, got) { + t.Fatal("Resample not working correctly") + } + }) } } } @@ -90,3 +93,22 @@ func lagrange(pts []point, x float64) (y float64) { type point struct { X, Y float64 } + +func FuzzResampler_SetRatio(f *testing.F) { + f.Add(44100, 48000, 0.5, 1.0, 8.0) + f.Fuzz(func(t *testing.T, original, desired int, r1, r2, r3 float64) { + if original <= 0 || desired <= 0 || r1 <= 0 || r2 <= 0 || r3 <= 0 { + t.Skip() + } + + s, _ := testtools.RandomDataStreamer(1e4) + r := beep.Resample(4, beep.SampleRate(original), beep.SampleRate(desired), s) + testtools.CollectNum(1024, r) + r.SetRatio(r1) + testtools.CollectNum(1024, r) + r.SetRatio(r2) + testtools.CollectNum(1024, r) + r.SetRatio(r3) + testtools.CollectNum(1024, r) + }) +} From 0705855c96e2259e494a8cb6669a88e9b47cce44 Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 19:46:13 +0200 Subject: [PATCH 6/8] Fix typo --- resample.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resample.go b/resample.go index 7958e99..d2deba6 100644 --- a/resample.go +++ b/resample.go @@ -68,7 +68,7 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { // changing of the resampling ratio, which can be useful for dynamically changing the speed of // streaming. type Resampler struct { - s Streamer // the orignal streamer + s Streamer // the original streamer ratio float64 // old sample rate / new sample rate buf1, buf2 [][2]float64 // buf1 contains previous buf2, new data goes into buf2, buf1 is because interpolation might require old samples pts []point // pts is for points used for interpolation From 6fbcee95148cb3e3265914d1be948a7061d78ac1 Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Thu, 11 Jul 2024 20:51:39 +0200 Subject: [PATCH 7/8] Limit resample ratio in speedy-player example --- examples/speedy-player/main.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/examples/speedy-player/main.go b/examples/speedy-player/main.go index 97bf5a7..bf199d6 100644 --- a/examples/speedy-player/main.go +++ b/examples/speedy-player/main.go @@ -7,6 +7,7 @@ import ( "unicode" "github.com/gdamore/tcell/v2" + "github.com/gopxl/beep" "github.com/gopxl/beep/effects" "github.com/gopxl/beep/mp3" @@ -128,13 +129,21 @@ func (ap *audioPanel) handle(event tcell.Event) (changed, quit bool) { case 'z': speaker.Lock() - ap.resampler.SetRatio(ap.resampler.Ratio() * 15 / 16) + newRatio := ap.resampler.Ratio() * 15 / 16 + if newRatio < 0.001 { + newRatio = 0.001 + } + ap.resampler.SetRatio(newRatio) speaker.Unlock() return true, false case 'x': speaker.Lock() - ap.resampler.SetRatio(ap.resampler.Ratio() * 16 / 15) + newRatio := ap.resampler.Ratio() * 16 / 15 + if newRatio > 100 { + newRatio = 100 + } + ap.resampler.SetRatio(newRatio) speaker.Unlock() return true, false } From df94d52c6faba396675a33ce2e4aeea612933bd0 Mon Sep 17 00:00:00 2001 From: Mark Kremer Date: Wed, 31 Jul 2024 15:16:20 +0200 Subject: [PATCH 8/8] Add explanatory comment for initial value of Resampler.off --- resample.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/resample.go b/resample.go index d2deba6..c2eeee5 100644 --- a/resample.go +++ b/resample.go @@ -58,9 +58,17 @@ func ResampleRatio(quality int, ratio float64, s Streamer) *Resampler { buf1: make([][2]float64, resamplerSingleBufferSize), buf2: make([][2]float64, resamplerSingleBufferSize), pts: make([]point, quality*2), - off: -resamplerSingleBufferSize, - pos: 0.0, - end: math.MaxInt, + // The initial value of `off` is set so that the current position is just behind the end + // of buf2: + // current position (0) - len(buf2) = -resamplerSingleBufferSize + // When the Stream() method is called for the first time, it will determine that neither + // buf1 nor buf2 contain the required samples because they are both in the past relative to + // the chosen `off` value. As a result, buf2 will be filled with samples, and `off` will be + // incremented by resamplerSingleBufferSize, making `off` equal to 0. This will align the + // start of buf2 with the current position. + off: -resamplerSingleBufferSize, + pos: 0.0, + end: math.MaxInt, } }