feat(BRIDGE-206): added observability service suport & metrics

builder.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ProtonMail/gluon/internal/db_impl/sqlite3"
 	"github.com/ProtonMail/gluon/internal/session"
 	"github.com/ProtonMail/gluon/limits"
+	"github.com/ProtonMail/gluon/observability"
 	"github.com/ProtonMail/gluon/profiling"
 	"github.com/ProtonMail/gluon/reporter"
 	"github.com/ProtonMail/gluon/store"
@@ -39,6 +40,7 @@ type serverBuilder struct {
 	uidValidityGenerator imap.UIDValidityGenerator
 	panicHandler         async.PanicHandler
 	dbCI                 db.ClientInterface
+	observabilitySender  observability.Sender
 }
 
 func newBuilder() (*serverBuilder, error) {
@@ -52,6 +54,7 @@ func newBuilder() (*serverBuilder, error) {
 		uidValidityGenerator: imap.DefaultEpochUIDValidityGenerator(),
 		panicHandler:         async.NoopPanicHandler{},
 		dbCI:                 sqlite3.NewBuilder(),
+		observabilitySender:  nil,
 	}, nil
 }
 
@@ -121,6 +124,7 @@ func (builder *serverBuilder) build() (*Server, error) {
 		disableParallelism:   builder.disableParallelism,
 		uidValidityGenerator: builder.uidValidityGenerator,
 		panicHandler:         builder.panicHandler,
+		observabilitySender:  builder.observabilitySender,
 	}
 
 	return s, nil

internal/backend/user.go

@@ -14,6 +14,8 @@ import (
 	"github.com/ProtonMail/gluon/internal/utils"
 	"github.com/ProtonMail/gluon/limits"
 	"github.com/ProtonMail/gluon/logging"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/reporter"
 	"github.com/ProtonMail/gluon/store"
 	"github.com/bradenaw/juniper/xslices"
@@ -149,6 +151,7 @@ func newUser(
 
 	if err := user.deleteAllMessagesMarkedDeleted(ctx); err != nil {
 		log.WithError(err).Error("Failed to remove deleted messages")
+		observability.AddMessageRelatedMetric(ctx, metrics.GenerateFailedToRemoveDeletedMessagesMetric())
 	}
 
 	if err := user.cleanupStaleStoreData(ctx); err != nil {

internal/db_impl/sqlite3/client.go

@@ -14,6 +14,8 @@ import (
 	"github.com/ProtonMail/gluon/db"
 	"github.com/ProtonMail/gluon/imap"
 	"github.com/ProtonMail/gluon/internal/db_impl/sqlite3/utils"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/google/uuid"
 	_ "github.com/mattn/go-sqlite3"
 	"github.com/sirupsen/logrus"
@@ -193,6 +195,7 @@ func (c *Client) wrapTx(ctx context.Context, op func(context.Context, *sql.Tx, *
 	if err := tx.Commit(); err != nil {
 		if c.debug {
 			entry.Debugf("Failed to commit Transaction")
+			observability.AddOtherMetric(ctx, metrics.GenerateFailedToCommitDatabaseTransactionMetric())
 		}
 
 		return fmt.Errorf("%v: %w", err, db.ErrTransactionFailed)

internal/session/command.go

@@ -9,6 +9,8 @@ import (
 	"github.com/ProtonMail/gluon/imap/command"
 	"github.com/ProtonMail/gluon/internal/response"
 	"github.com/ProtonMail/gluon/logging"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/rfcparser"
 )
 
@@ -62,6 +64,7 @@ func (s *Session) startCommandReader(ctx context.Context) <-chan commandResult {
 				}
 
 				s.log.WithError(err).WithField("type", parser.LastParsedCommand()).Error("Failed to parse IMAP command")
+				observability.AddImapMetric(ctx, metrics.GenerateFailedParseIMAPCommandMetric())
 			} else {
 				s.log.Debug(cmd.SanitizedString())
 			}

internal/session/handle_copy.go

@@ -8,6 +8,8 @@ import (
 	"github.com/ProtonMail/gluon/internal/contexts"
 	"github.com/ProtonMail/gluon/internal/response"
 	"github.com/ProtonMail/gluon/internal/state"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/profiling"
 )
 
@@ -37,6 +39,7 @@ func (s *Session) handleCopy(ctx context.Context, tag string, cmd *command.Copy,
 	} else if errors.Is(err, state.ErrNoSuchMailbox) {
 		return response.No(tag).WithError(err).WithItems(response.ItemTryCreate()), nil
 	} else if err != nil {
+		observability.AddMessageRelatedMetric(ctx, metrics.GenerateFailedToCopyMessagesMetric())
 		return nil, err
 	}
 

internal/session/handle_create.go

@@ -7,6 +7,8 @@ import (
 	"github.com/ProtonMail/gluon/imap"
 	"github.com/ProtonMail/gluon/imap/command"
 	"github.com/ProtonMail/gluon/internal/response"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/profiling"
 )
 
@@ -24,6 +26,7 @@ func (s *Session) handleCreate(ctx context.Context, tag string, cmd *command.Cre
 	}
 
 	if err := s.state.Create(ctx, nameUTF8); err != nil {
+		observability.AddMessageRelatedMetric(ctx, metrics.GenerateFailedToCreateMailbox())
 		return err
 	}
 

internal/session/handle_delete.go

@@ -7,6 +7,8 @@ import (
 	"github.com/ProtonMail/gluon/imap"
 	"github.com/ProtonMail/gluon/imap/command"
 	"github.com/ProtonMail/gluon/internal/response"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/profiling"
 )
 
@@ -25,6 +27,7 @@ func (s *Session) handleDelete(ctx context.Context, tag string, cmd *command.Del
 
 	selectedDeleted, err := s.state.Delete(ctx, nameUTF8)
 	if err != nil {
+		observability.AddOtherMetric(ctx, metrics.GenerateFailedToDeleteMailboxMetric())
 		return err
 	}
 

internal/session/handle_move.go

@@ -8,6 +8,8 @@ import (
 	"github.com/ProtonMail/gluon/internal/contexts"
 	"github.com/ProtonMail/gluon/internal/response"
 	"github.com/ProtonMail/gluon/internal/state"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/profiling"
 )
 
@@ -35,6 +37,7 @@ func (s *Session) handleMove(ctx context.Context, tag string, cmd *command.Move,
 	} else if errors.Is(err, state.ErrNoSuchMailbox) {
 		return response.No(tag).WithError(err).WithItems(response.ItemTryCreate()), nil
 	} else if err != nil {
+		observability.AddMessageRelatedMetric(ctx, metrics.GenerateFailedToMoveMessagesFromMailboxMetric())
 		return nil, err
 	}
 

internal/state/mailbox.go

@@ -13,6 +13,8 @@ import (
 	"github.com/ProtonMail/gluon/imap/command"
 	"github.com/ProtonMail/gluon/internal/ids"
 	"github.com/ProtonMail/gluon/internal/response"
+	"github.com/ProtonMail/gluon/observability"
+	"github.com/ProtonMail/gluon/observability/metrics"
 	"github.com/ProtonMail/gluon/rfc822"
 	"github.com/bradenaw/juniper/xslices"
 	"github.com/sirupsen/logrus"
@@ -250,6 +252,7 @@ func (m *Mailbox) Append(ctx context.Context, literal []byte, flags imap.FlagSet
 		})
 		if recoverErr != nil && !knownMessage {
 			m.log.WithError(recoverErr).Error("Failed to insert message into recovery mailbox")
+			observability.AddOtherMetric(ctx, metrics.GenerateFailedToInsertMessageIntoRecoveryMailbox())
 		}
 
 		if knownMessage {

observability/context.go

@@ -0,0 +1,21 @@
+package observability
+
+import "context"
+
+type observabilitySenderKeyType struct{}
+
+var observabilitySenderKeyVal observabilitySenderKeyType
+
+func NewContextWithObservabilitySender(ctx context.Context, sender Sender) context.Context {
+	return context.WithValue(ctx, observabilitySenderKeyVal, sender)
+}
+
+func getObservabilitySenderFromContext(ctx context.Context) (Sender, bool) {
+	v := ctx.Value(observabilitySenderKeyVal)
+	if v == nil {
+		return nil, false
+	}
+
+	sender, ok := v.(Sender)
+	return sender, ok
+}

observability/metrics/metrics.go

@@ -0,0 +1,52 @@
+package metrics
+
+import "time"
+
+const schemaName = "bridge_gluon_errors_total"
+const schemaVersion = 1
+
+func generateGluonFailureMetric(errorType string) map[string]interface{} {
+	return map[string]interface{}{
+		"Name":      schemaName,
+		"Version":   schemaVersion,
+		"Timestamp": time.Now().Unix(),
+		"Data": map[string]interface{}{
+			"Value": 1,
+			"Labels": map[string]string{
+				"errorType": errorType,
+			},
+		},
+	}
+}
+
+func GenerateFailedParseIMAPCommandMetric() map[string]interface{} {
+	return generateGluonFailureMetric("failedParseIMAPCommand")
+}
+
+func GenerateFailedToCreateMailbox() map[string]interface{} {
+	return generateGluonFailureMetric("failedCreateMailbox")
+}
+
+func GenerateFailedToDeleteMailboxMetric() map[string]interface{} {
+	return generateGluonFailureMetric("failedDeleteMailbox")
+}
+
+func GenerateFailedToCopyMessagesMetric() map[string]interface{} {
+	return generateGluonFailureMetric("failedCopyMessages")
+}
+
+func GenerateFailedToMoveMessagesFromMailboxMetric() map[string]interface{} {
+	return generateGluonFailureMetric("failedMoveMessagesFromMailbox")
+}
+
+func GenerateFailedToRemoveDeletedMessagesMetric() map[string]interface{} {
+	return generateGluonFailureMetric("failedRemoveDeletedMessages")
+}
+
+func GenerateFailedToCommitDatabaseTransactionMetric() map[string]interface{} {
+	return generateGluonFailureMetric("failedCommitDatabaseTransaction")
+}
+
+func GenerateFailedToInsertMessageIntoRecoveryMailbox() map[string]interface{} {
+	return generateGluonFailureMetric("failedToInsertMessageIntoRecoveryMailbox")
+}

observability/observability.go

@@ -0,0 +1,16 @@
+package observability
+
+var imapErrorMetricType int
+var messageErrorMetricType int
+var otherErrorMetricType int
+
+type Sender interface {
+	AddMetrics(metrics ...map[string]interface{})
+	AddDistinctMetrics(errType interface{}, metrics ...map[string]interface{})
+}
+
+func SetupMetricTypes(imapErrorType, messageErrorType, otherErrorType int) {
+	imapErrorMetricType = imapErrorType
+	messageErrorMetricType = messageErrorType
+	otherErrorMetricType = otherErrorType
+}

observability/utils.go

@@ -0,0 +1,27 @@
+package observability
+
+import "context"
+
+func AddImapMetric(ctx context.Context, metric ...map[string]interface{}) {
+	sender, ok := getObservabilitySenderFromContext(ctx)
+	if !ok {
+		return
+	}
+	sender.AddDistinctMetrics(imapErrorMetricType, metric...)
+}
+
+func AddMessageRelatedMetric(ctx context.Context, metric ...map[string]interface{}) {
+	sender, ok := getObservabilitySenderFromContext(ctx)
+	if !ok {
+		return
+	}
+	sender.AddDistinctMetrics(messageErrorMetricType, metric...)
+}
+
+func AddOtherMetric(ctx context.Context, metric ...map[string]interface{}) {
+	sender, ok := getObservabilitySenderFromContext(ctx)
+	if !ok {
+		return
+	}
+	sender.AddDistinctMetrics(otherErrorMetricType, metric...)
+}

option.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ProtonMail/gluon/db"
 	"github.com/ProtonMail/gluon/imap"
 	limits2 "github.com/ProtonMail/gluon/limits"
+	"github.com/ProtonMail/gluon/observability"
 	"github.com/ProtonMail/gluon/profiling"
 	"github.com/ProtonMail/gluon/reporter"
 	"github.com/ProtonMail/gluon/store"
@@ -243,3 +244,16 @@ func (w withDBClient) config(builder *serverBuilder) {
 func WithDBClient(ci db.ClientInterface) Option {
 	return &withDBClient{ci: ci}
 }
+
+type withObservabilitySender struct {
+	sender observability.Sender
+}
+
+func (w withObservabilitySender) config(builder *serverBuilder) {
+	builder.observabilitySender = w.sender
+}
+
+func WithObsSender(sender observability.Sender, imapErrorType, messageErrorType, otherErrorType int) Option {
+	observability.SetupMetricTypes(imapErrorType, messageErrorType, otherErrorType)
+	return &withObservabilitySender{sender: sender}
+}

server.go

@@ -19,6 +19,7 @@ import (
 	"github.com/ProtonMail/gluon/internal/contexts"
 	"github.com/ProtonMail/gluon/internal/session"
 	"github.com/ProtonMail/gluon/logging"
+	"github.com/ProtonMail/gluon/observability"
 	"github.com/ProtonMail/gluon/profiling"
 	"github.com/ProtonMail/gluon/reporter"
 	"github.com/ProtonMail/gluon/store"
@@ -88,6 +89,8 @@ type Server struct {
 	uidValidityGenerator imap.UIDValidityGenerator
 
 	panicHandler async.PanicHandler
+
+	observabilitySender observability.Sender
 }
 
 // New creates a new server with the given options.
@@ -172,6 +175,7 @@ func (s *Server) AddWatcher(ofType ...events.Event) <-chan events.Event {
 // Serve serves connections accepted from the given listener.
 // It stops serving when the context is canceled, the listener is closed, or the server is closed.
 func (s *Server) Serve(ctx context.Context, l net.Listener) error {
+	ctx = observability.NewContextWithObservabilitySender(ctx, s.observabilitySender)
 	ctx = reporter.NewContextWithReporter(ctx, s.reporter)
 	ctx = contexts.NewDisableParallelismCtx(ctx, s.disableParallelism)