From 144a692516e4598d941331b3b2a4321bf019ff31 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Tue, 30 Jul 2024 11:26:59 +0100 Subject: [PATCH 1/2] Give up on remaining analytics messages when we fail to push to the buffer. Log how many messages were in there --- handlers/analytics.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/handlers/analytics.go b/handlers/analytics.go index d775025f..036d4f75 100644 --- a/handlers/analytics.go +++ b/handlers/analytics.go @@ -123,13 +123,15 @@ func (c *AnalyticsHandlersCollection) Log() httprouter.Handle { cerrors.WriteHTTPBadRequest(w, "Invalid playback_id", nil) } - for _, ad := range c.toAnalyticsData(log, geo, extData) { + data := c.toAnalyticsData(log, geo, extData) + for i, ad := range data { select { case dataCh <- ad: // process data async default: - glog.Warningf("error processing analytics log, too many requests") + glog.Warningf("error processing analytics log, too many requests. Failed to write %d lines", len(data)-i) cerrors.WriteHTTPInternalServerError(w, "error processing analytics log, too many requests", nil) + return } } } From 381e58792cee53027f8e5a138072cc3502324756 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Tue, 30 Jul 2024 11:32:25 +0100 Subject: [PATCH 2/2] Build some more information about the message types we're overloaded with --- handlers/analytics.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/handlers/analytics.go b/handlers/analytics.go index 036d4f75..f6ffceea 100644 --- a/handlers/analytics.go +++ b/handlers/analytics.go @@ -129,7 +129,12 @@ func (c *AnalyticsHandlersCollection) Log() httprouter.Handle { case dataCh <- ad: // process data async default: - glog.Warningf("error processing analytics log, too many requests. Failed to write %d lines", len(data)-i) + // Do some counting up of the different message types to give us a better picture of what's going on here + msgTypes := map[string]int{} + for _, msg := range data { + msgTypes[msg.EventType] += 1 + } + glog.Warningf("error processing analytics log, too many requests. Failed to write %d lines. Message types: %v", len(data)-i, msgTypes) cerrors.WriteHTTPInternalServerError(w, "error processing analytics log, too many requests", nil) return }