diff --git a/.github/workflows/golang-lint.yml b/.github/workflows/golang-lint.yml index f358b496b8..aadcfa7a57 100644 --- a/.github/workflows/golang-lint.yml +++ b/.github/workflows/golang-lint.yml @@ -29,6 +29,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v6 with: - version: v1.60 + version: v1.61 working-directory: ./flow args: --timeout=10m diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index a54b44a43b..3e8aa81115 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -192,6 +192,9 @@ services: dockerfile: stacks/peerdb-ui.Dockerfile ports: - 3000:3000 + env_file: + - path: ./.env + required: false environment: <<: *catalog-config DATABASE_URL: postgres://postgres:postgres@catalog:5432/postgres diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go index 392b5c7671..93ae644513 100644 --- a/flow/activities/flowable.go +++ b/flow/activities/flowable.go @@ -361,19 +361,6 @@ func (a *FlowableActivity) StartNormalize( return nil, fmt.Errorf("failed to normalized records: %w", err) } - // normalize flow did not run due to no records, no need to update end time. - if res.Done { - err = monitoring.UpdateEndTimeForCDCBatch( - ctx, - a.CatalogPool, - input.FlowConnectionConfigs.FlowJobName, - res.EndBatchID, - ) - if err != nil { - return nil, err - } - } - // log the number of batches normalized logger.Info(fmt.Sprintf("normalized records from batch %d to batch %d", res.StartBatchID, res.EndBatchID)) @@ -702,7 +689,11 @@ func (a *FlowableActivity) RecordSlotSizes(ctx context.Context) error { slotMetricGauges.OpenReplicationConnectionsGauge = openReplicationConnectionsGauge } - if err := srcConn.HandleSlotInfo(ctx, a.Alerter, a.CatalogPool, slotName, peerName, slotMetricGauges); err != nil { + if err := srcConn.HandleSlotInfo(ctx, a.Alerter, a.CatalogPool, &alerting.AlertKeys{ + FlowName: config.FlowJobName, + PeerName: peerName, + SlotName: slotName, + }, slotMetricGauges); err != nil { logger.Error("Failed to handle slot info", slog.Any("error", err)) } }() diff --git a/flow/alerting/alerting.go b/flow/alerting/alerting.go index f75059c6ec..7c08c00fe1 100644 --- a/flow/alerting/alerting.go +++ b/flow/alerting/alerting.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "slices" "strings" "time" @@ -25,44 +26,59 @@ type Alerter struct { } type AlertSenderConfig struct { - Sender AlertSender - Id int64 + Sender AlertSender + AlertForMirrors []string + Id int64 +} + +type AlertKeys struct { + FlowName string + PeerName string + SlotName string } func (a *Alerter) registerSendersFromPool(ctx context.Context) ([]AlertSenderConfig, error) { rows, err := a.catalogPool.Query(ctx, - "SELECT id,service_type,service_config,enc_key_id FROM peerdb_stats.alerting_config") + `SELECT + id, + service_type, + service_config, + enc_key_id, + alert_for_mirrors + FROM peerdb_stats.alerting_config`) if err != nil { return nil, fmt.Errorf("failed to read alerter config from catalog: %w", err) } keys := peerdbenv.PeerDBEncKeys() return pgx.CollectRows(rows, func(row pgx.CollectableRow) (AlertSenderConfig, error) { - var id int64 + var alertSenderConfig AlertSenderConfig var serviceType ServiceType var serviceConfigEnc []byte var encKeyId string - if err := row.Scan(&id, &serviceType, &serviceConfigEnc, &encKeyId); err != nil { - return AlertSenderConfig{}, err + if err := row.Scan(&alertSenderConfig.Id, &serviceType, &serviceConfigEnc, &encKeyId, + &alertSenderConfig.AlertForMirrors); err != nil { + return alertSenderConfig, err } key, err := keys.Get(encKeyId) if err != nil { - return AlertSenderConfig{}, err + return alertSenderConfig, err } serviceConfig, err := key.Decrypt(serviceConfigEnc) if err != nil { - return AlertSenderConfig{}, err + return alertSenderConfig, err } switch serviceType { case SLACK: var slackServiceConfig slackAlertConfig if err := json.Unmarshal(serviceConfig, &slackServiceConfig); err != nil { - return AlertSenderConfig{}, fmt.Errorf("failed to unmarshal %s service config: %w", serviceType, err) + return alertSenderConfig, fmt.Errorf("failed to unmarshal %s service config: %w", serviceType, err) } - return AlertSenderConfig{Id: id, Sender: newSlackAlertSender(&slackServiceConfig)}, nil + alertSenderConfig.Sender = newSlackAlertSender(&slackServiceConfig) + return alertSenderConfig, nil case EMAIL: var replyToAddresses []string if replyToEnvString := strings.TrimSpace( @@ -75,10 +91,10 @@ func (a *Alerter) registerSendersFromPool(ctx context.Context) ([]AlertSenderCon replyToAddresses: replyToAddresses, } if emailServiceConfig.sourceEmail == "" { - return AlertSenderConfig{}, errors.New("missing sourceEmail for Email alerting service") + return alertSenderConfig, errors.New("missing sourceEmail for Email alerting service") } if err := json.Unmarshal(serviceConfig, &emailServiceConfig); err != nil { - return AlertSenderConfig{}, fmt.Errorf("failed to unmarshal %s service config: %w", serviceType, err) + return alertSenderConfig, fmt.Errorf("failed to unmarshal %s service config: %w", serviceType, err) } var region *string if envRegion := peerdbenv.PeerDBAlertingEmailSenderRegion(); envRegion != "" { @@ -89,9 +105,11 @@ func (a *Alerter) registerSendersFromPool(ctx context.Context) ([]AlertSenderCon if alertSenderErr != nil { return AlertSenderConfig{}, fmt.Errorf("failed to initialize email alerter: %w", alertSenderErr) } - return AlertSenderConfig{Id: id, Sender: alertSender}, nil + alertSenderConfig.Sender = alertSender + + return alertSenderConfig, nil default: - return AlertSenderConfig{}, fmt.Errorf("unknown service type: %s", serviceType) + return alertSenderConfig, fmt.Errorf("unknown service type: %s", serviceType) } }) } @@ -119,7 +137,7 @@ func NewAlerter(ctx context.Context, catalogPool *pgxpool.Pool) *Alerter { } } -func (a *Alerter) AlertIfSlotLag(ctx context.Context, peerName string, slotInfo *protos.SlotInfo) { +func (a *Alerter) AlertIfSlotLag(ctx context.Context, alertKeys *AlertKeys, slotInfo *protos.SlotInfo) { alertSenderConfigs, err := a.registerSendersFromPool(ctx) if err != nil { logger.LoggerFromCtx(ctx).Warn("failed to set alert senders", slog.Any("error", err)) @@ -144,12 +162,16 @@ func (a *Alerter) AlertIfSlotLag(ctx context.Context, peerName string, slotInfo } } - alertKey := fmt.Sprintf("%s Slot Lag Threshold Exceeded for Peer %s", deploymentUIDPrefix, peerName) + alertKey := fmt.Sprintf("%s Slot Lag Threshold Exceeded for Peer %s", deploymentUIDPrefix, alertKeys.PeerName) alertMessageTemplate := fmt.Sprintf("%sSlot `%s` on peer `%s` has exceeded threshold size of %%dMB, "+ - `currently at %.2fMB!`, deploymentUIDPrefix, slotInfo.SlotName, peerName, slotInfo.LagInMb) + `currently at %.2fMB!`, deploymentUIDPrefix, slotInfo.SlotName, alertKeys.PeerName, slotInfo.LagInMb) if slotInfo.LagInMb > float32(lowestSlotLagMBAlertThreshold) { for _, alertSenderConfig := range alertSenderConfigs { + if len(alertSenderConfig.AlertForMirrors) > 0 && + !slices.Contains(alertSenderConfig.AlertForMirrors, alertKeys.FlowName) { + continue + } if a.checkAndAddAlertToCatalog(ctx, alertSenderConfig.Id, alertKey, fmt.Sprintf(alertMessageTemplate, lowestSlotLagMBAlertThreshold)) { if alertSenderConfig.Sender.getSlotLagMBAlertThreshold() > 0 { @@ -168,7 +190,7 @@ func (a *Alerter) AlertIfSlotLag(ctx context.Context, peerName string, slotInfo } } -func (a *Alerter) AlertIfOpenConnections(ctx context.Context, peerName string, +func (a *Alerter) AlertIfOpenConnections(ctx context.Context, alertKeys *AlertKeys, openConnections *protos.GetOpenConnectionsForUserResult, ) { alertSenderConfigs, err := a.registerSendersFromPool(ctx) @@ -191,17 +213,22 @@ func (a *Alerter) AlertIfOpenConnections(ctx context.Context, peerName string, lowestOpenConnectionsThreshold := defaultOpenConnectionsThreshold for _, alertSender := range alertSenderConfigs { if alertSender.Sender.getOpenConnectionsAlertThreshold() > 0 { - lowestOpenConnectionsThreshold = min(lowestOpenConnectionsThreshold, alertSender.Sender.getOpenConnectionsAlertThreshold()) + lowestOpenConnectionsThreshold = min(lowestOpenConnectionsThreshold, + alertSender.Sender.getOpenConnectionsAlertThreshold()) } } - alertKey := fmt.Sprintf("%s Max Open Connections Threshold Exceeded for Peer %s", deploymentUIDPrefix, peerName) + alertKey := fmt.Sprintf("%s Max Open Connections Threshold Exceeded for Peer %s", deploymentUIDPrefix, alertKeys.PeerName) alertMessageTemplate := fmt.Sprintf("%sOpen connections from PeerDB user `%s` on peer `%s`"+ ` has exceeded threshold size of %%d connections, currently at %d connections!`, - deploymentUIDPrefix, openConnections.UserName, peerName, openConnections.CurrentOpenConnections) + deploymentUIDPrefix, openConnections.UserName, alertKeys.PeerName, openConnections.CurrentOpenConnections) if openConnections.CurrentOpenConnections > int64(lowestOpenConnectionsThreshold) { for _, alertSenderConfig := range alertSenderConfigs { + if len(alertSenderConfig.AlertForMirrors) > 0 && + !slices.Contains(alertSenderConfig.AlertForMirrors, alertKeys.FlowName) { + continue + } if a.checkAndAddAlertToCatalog(ctx, alertSenderConfig.Id, alertKey, fmt.Sprintf(alertMessageTemplate, lowestOpenConnectionsThreshold)) { if alertSenderConfig.Sender.getOpenConnectionsAlertThreshold() > 0 { diff --git a/flow/cmd/alerts.go b/flow/cmd/alerts.go index d3ea2b1c72..863aeab794 100644 --- a/flow/cmd/alerts.go +++ b/flow/cmd/alerts.go @@ -11,7 +11,7 @@ import ( ) func (h *FlowRequestHandler) GetAlertConfigs(ctx context.Context, req *protos.GetAlertConfigsRequest) (*protos.GetAlertConfigsResponse, error) { - rows, err := h.pool.Query(ctx, "select id, service_type, service_config, enc_key_id from peerdb_stats.alerting_config") + rows, err := h.pool.Query(ctx, "SELECT id,service_type,service_config,enc_key_id,alert_for_mirrors from peerdb_stats.alerting_config") if err != nil { return nil, err } @@ -20,7 +20,7 @@ func (h *FlowRequestHandler) GetAlertConfigs(ctx context.Context, req *protos.Ge var serviceConfigPayload []byte var encKeyID string config := &protos.AlertConfig{} - if err := row.Scan(&config.Id, &config.ServiceType, &serviceConfigPayload, &encKeyID); err != nil { + if err := row.Scan(&config.Id, &config.ServiceType, &serviceConfigPayload, &encKeyID, &config.AlertForMirrors); err != nil { return nil, err } serviceConfig, err := peerdbenv.Decrypt(encKeyID, serviceConfigPayload) @@ -42,34 +42,46 @@ func (h *FlowRequestHandler) PostAlertConfig(ctx context.Context, req *protos.Po if err != nil { return nil, err } - serviceConfig, err := key.Encrypt(shared.UnsafeFastStringToReadOnlyBytes(req.ServiceConfig)) + serviceConfig, err := key.Encrypt(shared.UnsafeFastStringToReadOnlyBytes(req.Config.ServiceConfig)) if err != nil { return nil, err } - if req.Id == -1 { + if req.Config.Id == -1 { var id int32 if err := h.pool.QueryRow( ctx, - "insert into peerdb_stats.alerting_config (service_type, service_config, enc_key_id) values ($1, $2, $3) returning id", - req.ServiceType, + `INSERT INTO peerdb_stats.alerting_config ( + service_type, + service_config, + enc_key_id, + alert_for_mirrors + ) VALUES ( + $1, + $2, + $3, + $4 + ) RETURNING id`, + req.Config.ServiceType, serviceConfig, key.ID, + req.Config.AlertForMirrors, ).Scan(&id); err != nil { return nil, err } return &protos.PostAlertConfigResponse{Id: id}, nil } else if _, err := h.pool.Exec( ctx, - "update peerdb_stats.alerting_config set service_type = $1, service_config = $2, enc_key_id = $3 where id = $4", - req.ServiceType, + "update peerdb_stats.alerting_config set service_type = $1, service_config = $2, enc_key_id = $3, alert_for_mirrors = $4 where id = $5", + req.Config.ServiceType, serviceConfig, key.ID, - req.Id, + req.Config.AlertForMirrors, + req.Config.Id, ); err != nil { return nil, err } - return &protos.PostAlertConfigResponse{Id: req.Id}, nil + return &protos.PostAlertConfigResponse{Id: req.Config.Id}, nil } func (h *FlowRequestHandler) DeleteAlertConfig( diff --git a/flow/connectors/bigquery/qrep_avro_sync.go b/flow/connectors/bigquery/qrep_avro_sync.go index da3b15c37f..f1f8d812c6 100644 --- a/flow/connectors/bigquery/qrep_avro_sync.go +++ b/flow/connectors/bigquery/qrep_avro_sync.go @@ -408,12 +408,12 @@ func (s *QRepAvroSyncMethod) writeToStage( obj := bucket.Object(avroFilePath) w := obj.NewWriter(ctx) - numRecords, err := ocfWriter.WriteOCF(ctx, w) + numRecords, err := ocfWriter.WriteOCF(ctx, w, -1) if err != nil { return 0, fmt.Errorf("failed to write records to Avro file on GCS: %w", err) } avroFile = &avro.AvroFile{ - NumRecords: numRecords, + NumRecords: int(numRecords), StorageLocation: avro.AvroGCSStorage, FilePath: avroFilePath, } diff --git a/flow/connectors/clickhouse/cdc.go b/flow/connectors/clickhouse/cdc.go index 56d04a0b6a..804b9d650f 100644 --- a/flow/connectors/clickhouse/cdc.go +++ b/flow/connectors/clickhouse/cdc.go @@ -66,13 +66,13 @@ func (c *ClickhouseConnector) CreateRawTable(ctx context.Context, req *protos.Cr }, nil } -func (c *ClickhouseConnector) avroSyncMethod(flowJobName string) *ClickhouseAvroSyncMethod { +func (c *ClickhouseConnector) avroSyncMethod(flowJobName string, env map[string]string) *ClickhouseAvroSyncMethod { qrepConfig := &protos.QRepConfig{ StagingPath: c.credsProvider.BucketPath, FlowJobName: flowJobName, DestinationTableIdentifier: c.getRawTableName(flowJobName), } - return NewClickhouseAvroSyncMethod(qrepConfig, c) + return NewClickhouseAvroSyncMethod(qrepConfig, c, env) } func (c *ClickhouseConnector) syncRecordsViaAvro( @@ -87,7 +87,7 @@ func (c *ClickhouseConnector) syncRecordsViaAvro( return nil, fmt.Errorf("failed to convert records to raw table stream: %w", err) } - avroSyncer := c.avroSyncMethod(req.FlowJobName) + avroSyncer := c.avroSyncMethod(req.FlowJobName, req.Env) numRecords, err := avroSyncer.SyncRecords(ctx, stream, req.FlowJobName, syncBatchID) if err != nil { return nil, err diff --git a/flow/connectors/clickhouse/normalize.go b/flow/connectors/clickhouse/normalize.go index 39fb5c9bd4..77b459f7f1 100644 --- a/flow/connectors/clickhouse/normalize.go +++ b/flow/connectors/clickhouse/normalize.go @@ -11,6 +11,7 @@ import ( "strings" "time" + utils "github.com/PeerDB-io/peer-flow/connectors/utils/avro" "github.com/PeerDB-io/peer-flow/datatypes" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" @@ -237,7 +238,7 @@ func (c *ClickhouseConnector) NormalizeRecords( }, nil } - err = c.copyAvroStagesToDestination(ctx, req.FlowJobName, normBatchID, req.SyncBatchID) + err = c.copyAvroStagesToDestination(ctx, req.FlowJobName, normBatchID, req.SyncBatchID, req.Env) if err != nil { return nil, fmt.Errorf("failed to copy avro stages to destination: %w", err) } @@ -344,8 +345,6 @@ func (c *ClickhouseConnector) NormalizeRecords( selectQuery.WriteString(tbl) selectQuery.WriteString("'") - selectQuery.WriteString(" ORDER BY _peerdb_timestamp") - insertIntoSelectQuery := strings.Builder{} insertIntoSelectQuery.WriteString("INSERT INTO ") insertIntoSelectQuery.WriteString(tbl) @@ -411,26 +410,28 @@ func (c *ClickhouseConnector) getDistinctTableNamesInBatch( return tableNames, nil } -func (c *ClickhouseConnector) copyAvroStageToDestination(ctx context.Context, flowJobName string, syncBatchID int64) error { - avroSynvMethod := c.avroSyncMethod(flowJobName) +func (c *ClickhouseConnector) copyAvroStageToDestination(ctx context.Context, + flowJobName string, syncBatchID int64, env map[string]string, +) error { + avroSyncMethod := c.avroSyncMethod(flowJobName, env) avroFile, err := c.s3Stage.GetAvroStage(ctx, flowJobName, syncBatchID) if err != nil { return fmt.Errorf("failed to get avro stage: %w", err) } defer avroFile.Cleanup() - err = avroSynvMethod.CopyStageToDestination(ctx, avroFile) + err = avroSyncMethod.CopyStageToDestination(ctx, []*utils.AvroFile{avroFile}) if err != nil { return fmt.Errorf("failed to copy stage to destination: %w", err) } return nil } -func (c *ClickhouseConnector) copyAvroStagesToDestination( - ctx context.Context, flowJobName string, normBatchID, syncBatchID int64, +func (c *ClickhouseConnector) copyAvroStagesToDestination(ctx context.Context, + flowJobName string, normBatchID, syncBatchID int64, env map[string]string, ) error { for s := normBatchID + 1; s <= syncBatchID; s++ { - err := c.copyAvroStageToDestination(ctx, flowJobName, s) + err := c.copyAvroStageToDestination(ctx, flowJobName, s, env) if err != nil { return fmt.Errorf("failed to copy avro stage to destination: %w", err) } diff --git a/flow/connectors/clickhouse/qrep.go b/flow/connectors/clickhouse/qrep.go index 105e1ee680..925e870a04 100644 --- a/flow/connectors/clickhouse/qrep.go +++ b/flow/connectors/clickhouse/qrep.go @@ -39,7 +39,7 @@ func (c *ClickhouseConnector) SyncQRepRecords( } c.logger.Info("Called QRep sync function and obtained table schema", flowLog) - avroSync := NewClickhouseAvroSyncMethod(config, c) + avroSync := NewClickhouseAvroSyncMethod(config, c, config.Env) return avroSync.SyncQRepRecords(ctx, config, partition, tblSchema, stream) } diff --git a/flow/connectors/clickhouse/qrep_avro_sync.go b/flow/connectors/clickhouse/qrep_avro_sync.go index b7288bf566..e3e12294f0 100644 --- a/flow/connectors/clickhouse/qrep_avro_sync.go +++ b/flow/connectors/clickhouse/qrep_avro_sync.go @@ -14,25 +14,29 @@ import ( "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/model" "github.com/PeerDB-io/peer-flow/model/qvalue" + "github.com/PeerDB-io/peer-flow/peerdbenv" "github.com/PeerDB-io/peer-flow/shared" ) type ClickhouseAvroSyncMethod struct { config *protos.QRepConfig connector *ClickhouseConnector + env map[string]string } func NewClickhouseAvroSyncMethod( config *protos.QRepConfig, connector *ClickhouseConnector, + env map[string]string, ) *ClickhouseAvroSyncMethod { return &ClickhouseAvroSyncMethod{ config: config, connector: connector, + env: env, } } -func (s *ClickhouseAvroSyncMethod) CopyStageToDestination(ctx context.Context, avroFile *avro.AvroFile) error { +func (s *ClickhouseAvroSyncMethod) CopyStageToDestination(ctx context.Context, avroFiles []*avro.AvroFile) error { stagingPath := s.connector.credsProvider.BucketPath s3o, err := utils.NewS3BucketAndPrefix(stagingPath) if err != nil { @@ -41,21 +45,30 @@ func (s *ClickhouseAvroSyncMethod) CopyStageToDestination(ctx context.Context, a endpoint := s.connector.credsProvider.Provider.GetEndpointURL() region := s.connector.credsProvider.Provider.GetRegion() - avroFileUrl := utils.FileURLForS3Service(endpoint, region, s3o.Bucket, avroFile.FilePath) creds, err := s.connector.credsProvider.Provider.Retrieve(ctx) if err != nil { return err } - sessionTokenPart := "" - if creds.AWS.SessionToken != "" { - sessionTokenPart = fmt.Sprintf(", '%s'", creds.AWS.SessionToken) + for _, avroFile := range avroFiles { + avroFileUrl := utils.FileURLForS3Service(endpoint, region, s3o.Bucket, avroFile.FilePath) + sessionTokenPart := "" + if creds.AWS.SessionToken != "" { + sessionTokenPart = fmt.Sprintf(", '%s'", creds.AWS.SessionToken) + } + + query := fmt.Sprintf("INSERT INTO %s SELECT * FROM s3('%s','%s','%s'%s, 'Avro')", + s.config.DestinationTableIdentifier, avroFileUrl, + creds.AWS.AccessKeyID, creds.AWS.SecretAccessKey, sessionTokenPart) + + s.connector.logger.Info("copying avro file to destination", slog.String("query", query)) + if err := s.connector.database.Exec(ctx, query); err != nil { + return err + } + s.connector.logger.Info("avro file copied to destination") } - query := fmt.Sprintf("INSERT INTO %s SELECT * FROM s3('%s','%s','%s'%s, 'Avro')", - s.config.DestinationTableIdentifier, avroFileUrl, - creds.AWS.AccessKeyID, creds.AWS.SecretAccessKey, sessionTokenPart) - return s.connector.database.Exec(ctx, query) + return nil } func (s *ClickhouseAvroSyncMethod) SyncRecords( @@ -75,24 +88,38 @@ func (s *ClickhouseAvroSyncMethod) SyncRecords( return 0, err } - batchIdentifierForFile := fmt.Sprintf("%s_%d", shared.RandomString(16), syncBatchID) - avroFile, err := s.writeToAvroFile(ctx, stream, avroSchema, batchIdentifierForFile, flowJobName) + partitionID := shared.RandomString(16) + avroFiles, err := s.writeToAvroFiles(ctx, stream, avroSchema, partitionID, flowJobName) if err != nil { return 0, err } + defer func() { + for _, avroFile := range avroFiles { + avroFile.Cleanup() + } + }() + totalRecords := 0 + for _, avroFile := range avroFiles { + err = s.connector.s3Stage.SetAvroStage(ctx, flowJobName, syncBatchID, avroFile) + if err != nil { + return 0, fmt.Errorf("failed to set avro stage: %w", err) + } + totalRecords += avroFile.NumRecords + } - s.connector.logger.Info("[SyncRecords] written records to Avro file", - slog.String("dstTable", dstTableName), - slog.String("avroFile", avroFile.FilePath), - slog.Int("numRecords", avroFile.NumRecords), - slog.Int64("syncBatchID", syncBatchID)) - - err = s.connector.s3Stage.SetAvroStage(ctx, flowJobName, syncBatchID, avroFile) + s.connector.logger.Info(fmt.Sprintf("written %d records to Avro files", totalRecords), + slog.String("dstTable", dstTableName)) + err = s.CopyStageToDestination(ctx, avroFiles) if err != nil { - return 0, fmt.Errorf("failed to set avro stage: %w", err) + return 0, err } - return avroFile.NumRecords, nil + s.connector.logger.Info("[SyncRecords] written records to Avro files", + slog.String("dstTable", dstTableName), + slog.Int("numRecords", totalRecords), + slog.Int64("syncBatchID", syncBatchID)) + + return totalRecords, nil } func (s *ClickhouseAvroSyncMethod) SyncQRepRecords( @@ -111,10 +138,14 @@ func (s *ClickhouseAvroSyncMethod) SyncQRepRecords( return 0, err } - avroFile, err := s.writeToAvroFile(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName) + avroFiles, err := s.writeToAvroFiles(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName) if err != nil { return 0, err } + totalRecords := 0 + for _, avroFile := range avroFiles { + totalRecords += avroFile.NumRecords + } s3o, err := utils.NewS3BucketAndPrefix(stagingPath) if err != nil { @@ -128,7 +159,6 @@ func (s *ClickhouseAvroSyncMethod) SyncQRepRecords( endpoint := s.connector.credsProvider.Provider.GetEndpointURL() region := s.connector.credsProvider.Provider.GetRegion() - avroFileUrl := utils.FileURLForS3Service(endpoint, region, s3o.Bucket, avroFile.FilePath) selector := make([]string, 0, len(dstTableSchema)) for _, col := range dstTableSchema { colName := col.Name() @@ -147,14 +177,18 @@ func (s *ClickhouseAvroSyncMethod) SyncQRepRecords( if creds.AWS.SessionToken != "" { sessionTokenPart = fmt.Sprintf(", '%s'", creds.AWS.SessionToken) } - query := fmt.Sprintf("INSERT INTO %s(%s) SELECT %s FROM s3('%s','%s','%s'%s, 'Avro')", - config.DestinationTableIdentifier, selectorStr, selectorStr, avroFileUrl, - creds.AWS.AccessKeyID, creds.AWS.SecretAccessKey, sessionTokenPart) - err = s.connector.database.Exec(ctx, query) - if err != nil { - s.connector.logger.Error("Failed to insert into select for Clickhouse: ", err) - return 0, err + for _, avroFile := range avroFiles { + avroFileUrl := utils.FileURLForS3Service(endpoint, region, s3o.Bucket, avroFile.FilePath) + + query := fmt.Sprintf("INSERT INTO %s(%s) SELECT %s FROM s3('%s','%s','%s'%s, 'Avro')", + config.DestinationTableIdentifier, selectorStr, selectorStr, avroFileUrl, + creds.AWS.AccessKeyID, creds.AWS.SecretAccessKey, sessionTokenPart) + + if err := s.connector.database.Exec(ctx, query); err != nil { + s.connector.logger.Error("Failed to insert into select for Clickhouse: ", err) + return 0, err + } } err = s.insertMetadata(ctx, partition, config.FlowJobName, startTime) @@ -162,7 +196,7 @@ func (s *ClickhouseAvroSyncMethod) SyncQRepRecords( return -1, err } - return avroFile.NumRecords, nil + return totalRecords, nil } func (s *ClickhouseAvroSyncMethod) getAvroSchema( @@ -176,13 +210,13 @@ func (s *ClickhouseAvroSyncMethod) getAvroSchema( return avroSchema, nil } -func (s *ClickhouseAvroSyncMethod) writeToAvroFile( +func (s *ClickhouseAvroSyncMethod) writeToAvroFiles( ctx context.Context, stream *model.QRecordStream, avroSchema *model.QRecordAvroSchemaDefinition, identifierForFile string, flowJobName string, -) (*avro.AvroFile, error) { +) ([]*avro.AvroFile, error) { stagingPath := s.connector.credsProvider.BucketPath ocfWriter := avro.NewPeerDBOCFWriter(stream, avroSchema, avro.CompressZstd, protos.DBType_CLICKHOUSE) s3o, err := utils.NewS3BucketAndPrefix(stagingPath) @@ -192,12 +226,22 @@ func (s *ClickhouseAvroSyncMethod) writeToAvroFile( s3AvroFileKey := fmt.Sprintf("%s/%s/%s.avro.zst", s3o.Prefix, flowJobName, identifierForFile) s3AvroFileKey = strings.Trim(s3AvroFileKey, "/") - avroFile, err := ocfWriter.WriteRecordsToS3(ctx, s3o.Bucket, s3AvroFileKey, s.connector.credsProvider.Provider) + maxRecordsPerFile, err := peerdbenv.PeerDBClickhouseNumRowsPerAvroFile(ctx, s.env) + if err != nil { + return nil, fmt.Errorf("failed to get max records per file: %w", err) + } + avroFiles, err := ocfWriter.WriteRecordsToS3Parts( + ctx, s3o.Bucket, s3AvroFileKey, s.connector.credsProvider.Provider, int64(maxRecordsPerFile)) if err != nil { return nil, fmt.Errorf("failed to write records to S3: %w", err) } - return avroFile, nil + s.connector.logger.Info("avro files written to S3", slog.Int("numFiles", len(avroFiles))) + for _, avroFile := range avroFiles { + s.connector.logger.Info("avro file written to S3", slog.String("filePath", avroFile.FilePath)) + } + + return avroFiles, nil } func (s *ClickhouseAvroSyncMethod) insertMetadata( diff --git a/flow/connectors/core.go b/flow/connectors/core.go index 0b7a35b1c2..e19693c7d4 100644 --- a/flow/connectors/core.go +++ b/flow/connectors/core.go @@ -79,8 +79,7 @@ type CDCPullConnectorCore interface { ctx context.Context, alerter *alerting.Alerter, catalogPool *pgxpool.Pool, - slotName string, - peerName string, + alertKeys *alerting.AlertKeys, slotMetricGauges peerdb_gauges.SlotMetricGauges, ) error diff --git a/flow/connectors/external_metadata/store.go b/flow/connectors/external_metadata/store.go index e591259512..b586ea3053 100644 --- a/flow/connectors/external_metadata/store.go +++ b/flow/connectors/external_metadata/store.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/sdk/log" "google.golang.org/protobuf/encoding/protojson" + "github.com/PeerDB-io/peer-flow/connectors/utils/monitoring" "github.com/PeerDB-io/peer-flow/generated/protos" "github.com/PeerDB-io/peer-flow/logger" "github.com/PeerDB-io/peer-flow/peerdbenv" @@ -179,6 +180,12 @@ func (p *PostgresMetadata) UpdateNormalizeBatchID(ctx context.Context, jobName s return err } + err = monitoring.UpdateEndTimeForCDCBatch(ctx, p.pool, jobName, batchID) + if err != nil { + p.logger.Error(fmt.Sprintf("failed to update end time for cdc batch - %d", batchID), slog.Any("error", err)) + return err + } + return nil } diff --git a/flow/connectors/postgres/cdc.go b/flow/connectors/postgres/cdc.go index dea7cae4cd..255631cf7d 100644 --- a/flow/connectors/postgres/cdc.go +++ b/flow/connectors/postgres/cdc.go @@ -218,7 +218,7 @@ func (p *PostgresCDCSource) decodeColumnData(data []byte, dataType uint32, forma var parsedData any var err error if dt, ok := p.typeMap.TypeForOID(dataType); ok { - if dt.Name == "uuid" || dt.Name == "cidr" || dt.Name == "inet" || dt.Name == "macaddr" { + if dt.Name == "uuid" || dt.Name == "cidr" || dt.Name == "inet" || dt.Name == "macaddr" || dt.Name == "xml" { // below is required to decode above types to string parsedData, err = dt.Codec.DecodeDatabaseSQLValue(p.typeMap, dataType, pgtype.TextFormatCode, data) } else { diff --git a/flow/connectors/postgres/postgres.go b/flow/connectors/postgres/postgres.go index b08d97837b..c54ad06024 100644 --- a/flow/connectors/postgres/postgres.go +++ b/flow/connectors/postgres/postgres.go @@ -1161,28 +1161,29 @@ func (c *PostgresConnector) HandleSlotInfo( ctx context.Context, alerter *alerting.Alerter, catalogPool *pgxpool.Pool, - slotName string, - peerName string, + alertKeys *alerting.AlertKeys, slotMetricGauges peerdb_gauges.SlotMetricGauges, ) error { logger := logger.LoggerFromCtx(ctx) - slotInfo, err := getSlotInfo(ctx, c.conn, slotName, c.config.Database) + slotInfo, err := getSlotInfo(ctx, c.conn, alertKeys.SlotName, c.config.Database) if err != nil { logger.Warn("warning: failed to get slot info", "error", err) return err } if len(slotInfo) == 0 { - logger.Warn("warning: unable to get slot info", "slotName", slotName) + logger.Warn("warning: unable to get slot info", slog.String("slotName", alertKeys.SlotName)) return nil } - logger.Info(fmt.Sprintf("Checking %s lag for %s", slotName, peerName), slog.Float64("LagInMB", float64(slotInfo[0].LagInMb))) - alerter.AlertIfSlotLag(ctx, peerName, slotInfo[0]) + logger.Info(fmt.Sprintf("Checking %s lag for %s", alertKeys.SlotName, alertKeys.PeerName), + slog.Float64("LagInMB", float64(slotInfo[0].LagInMb))) + alerter.AlertIfSlotLag(ctx, alertKeys, slotInfo[0]) slotMetricGauges.SlotLagGauge.Set(float64(slotInfo[0].LagInMb), attribute.NewSet( - attribute.String(peerdb_gauges.PeerNameKey, peerName), - attribute.String(peerdb_gauges.SlotNameKey, slotName), + attribute.String(peerdb_gauges.FlowNameKey, alertKeys.FlowName), + attribute.String(peerdb_gauges.PeerNameKey, alertKeys.PeerName), + attribute.String(peerdb_gauges.SlotNameKey, alertKeys.SlotName), attribute.String(peerdb_gauges.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID()))) // Also handles alerts for PeerDB user connections exceeding a given limit here @@ -1191,9 +1192,10 @@ func (c *PostgresConnector) HandleSlotInfo( logger.Warn("warning: failed to get current open connections", "error", err) return err } - alerter.AlertIfOpenConnections(ctx, peerName, res) + alerter.AlertIfOpenConnections(ctx, alertKeys, res) slotMetricGauges.OpenConnectionsGauge.Set(res.CurrentOpenConnections, attribute.NewSet( - attribute.String(peerdb_gauges.PeerNameKey, peerName), + attribute.String(peerdb_gauges.FlowNameKey, alertKeys.FlowName), + attribute.String(peerdb_gauges.PeerNameKey, alertKeys.PeerName), attribute.String(peerdb_gauges.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID()))) replicationRes, err := getOpenReplicationConnectionsForUser(ctx, c.conn, c.config.User) @@ -1203,10 +1205,11 @@ func (c *PostgresConnector) HandleSlotInfo( } slotMetricGauges.OpenReplicationConnectionsGauge.Set(replicationRes.CurrentOpenConnections, attribute.NewSet( - attribute.String(peerdb_gauges.PeerNameKey, peerName), + attribute.String(peerdb_gauges.FlowNameKey, alertKeys.FlowName), + attribute.String(peerdb_gauges.PeerNameKey, alertKeys.PeerName), attribute.String(peerdb_gauges.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID()))) - return monitoring.AppendSlotSizeInfo(ctx, catalogPool, peerName, slotInfo[0]) + return monitoring.AppendSlotSizeInfo(ctx, catalogPool, alertKeys.PeerName, slotInfo[0]) } func getOpenConnectionsForUser(ctx context.Context, conn *pgx.Conn, user string) (*protos.GetOpenConnectionsForUserResult, error) { diff --git a/flow/connectors/utils/avro/avro_writer.go b/flow/connectors/utils/avro/avro_writer.go index 6c78a04f7f..478535301c 100644 --- a/flow/connectors/utils/avro/avro_writer.go +++ b/flow/connectors/utils/avro/avro_writer.go @@ -127,7 +127,7 @@ func (p *peerDBOCFWriter) createOCFWriter(w io.Writer) (*goavro.OCFWriter, error return ocfWriter, nil } -func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter *goavro.OCFWriter) (int64, error) { +func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter *goavro.OCFWriter, maxRecords int64) (int64, error) { logger := logger.LoggerFromCtx(ctx) schema := p.stream.Schema() @@ -162,6 +162,9 @@ func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter } numRows.Add(1) + if maxRecords > 0 && numRows.Load() >= maxRecords { + break + } } } @@ -173,7 +176,7 @@ func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter return numRows.Load(), nil } -func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer) (int, error) { +func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer, maxRecords int64) (int64, error) { ocfWriter, err := p.createOCFWriter(w) if err != nil { return 0, fmt.Errorf("failed to create OCF writer: %w", err) @@ -181,16 +184,16 @@ func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer) (int, error // we have to keep a reference to the underlying writer as goavro doesn't provide any access to it defer p.writer.Close() - numRows, err := p.writeRecordsToOCFWriter(ctx, ocfWriter) + numRows, err := p.writeRecordsToOCFWriter(ctx, ocfWriter, maxRecords) if err != nil { return 0, fmt.Errorf("failed to write records to OCF writer: %w", err) } - return int(numRows), nil + return numRows, nil } -func (p *peerDBOCFWriter) WriteRecordsToS3( - ctx context.Context, bucketName, key string, s3Creds utils.AWSCredentialsProvider, -) (*AvroFile, error) { +func (p *peerDBOCFWriter) WriteRecordsToS3Parts( + ctx context.Context, bucketName, keyPrefix string, s3Creds utils.AWSCredentialsProvider, maxRecordsPerFile int64, +) ([]*AvroFile, error) { logger := logger.LoggerFromCtx(ctx) s3svc, err := utils.CreateS3Client(ctx, s3Creds) if err != nil { @@ -198,46 +201,70 @@ func (p *peerDBOCFWriter) WriteRecordsToS3( return nil, fmt.Errorf("failed to create S3 client: %w", err) } - buf := buffer.New(32 * 1024 * 1024) // 32MB in memory Buffer - r, w := nio.Pipe(buf) + var avroFiles []*AvroFile + part := 0 + for { + buf := buffer.New(32 * 1024 * 1024) // 32MB in memory Buffer + r, w := nio.Pipe(buf) + + var writeOcfError error + var numRows int64 + + go func() { + defer func() { + if r := recover(); r != nil { + writeOcfError = fmt.Errorf("panic occurred during WriteOCF: %v", r) + stack := string(debug.Stack()) + logger.Error("panic during WriteOCF", slog.Any("error", writeOcfError), slog.String("stack", stack)) + } + w.Close() + }() + numRows, writeOcfError = p.WriteOCF(ctx, w, maxRecordsPerFile) + }() - defer r.Close() - var writeOcfError error - var numRows int + key := fmt.Sprintf("%s_part_%d.avro.zst", keyPrefix, part) + _, err = manager.NewUploader(s3svc).Upload(ctx, &s3.PutObjectInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + Body: r, + }) + if err != nil { + s3Path := "s3://" + bucketName + "/" + key + logger.Error("failed to upload file: ", slog.Any("error", err), slog.Any("s3_path", s3Path)) + return nil, fmt.Errorf("failed to upload file to path %s: %w", s3Path, err) + } - go func() { - defer func() { - if r := recover(); r != nil { - writeOcfError = fmt.Errorf("panic occurred during WriteOCF: %v", r) - stack := string(debug.Stack()) - logger.Error("panic during WriteOCF", slog.Any("error", writeOcfError), slog.String("stack", stack)) - } - w.Close() - }() - numRows, writeOcfError = p.WriteOCF(ctx, w) - }() + if writeOcfError != nil { + logger.Error("failed to write records to OCF: ", slog.Any("error", writeOcfError)) + return nil, writeOcfError + } - _, err = manager.NewUploader(s3svc).Upload(ctx, &s3.PutObjectInput{ - Bucket: aws.String(bucketName), - Key: aws.String(key), - Body: r, - }) - if err != nil { - s3Path := "s3://" + bucketName + "/" + key - logger.Error("failed to upload file: ", slog.Any("error", err), slog.Any("s3_path", s3Path)) - return nil, fmt.Errorf("failed to upload file to path %s: %w", s3Path, err) - } + avroFiles = append(avroFiles, &AvroFile{ + NumRecords: int(numRows), + StorageLocation: AvroS3Storage, + FilePath: key, + }) - if writeOcfError != nil { - logger.Error("failed to write records to OCF: ", slog.Any("error", writeOcfError)) - return nil, writeOcfError + if maxRecordsPerFile < 0 || numRows < maxRecordsPerFile { + break + } + part++ } - return &AvroFile{ - NumRecords: numRows, - StorageLocation: AvroS3Storage, - FilePath: key, - }, nil + return avroFiles, nil +} + +func (p *peerDBOCFWriter) WriteRecordsToS3( + ctx context.Context, bucketName, key string, s3Creds utils.AWSCredentialsProvider, +) (*AvroFile, error) { + avroFiles, err := p.WriteRecordsToS3Parts(ctx, bucketName, key, s3Creds, -1) + if err != nil { + return nil, err + } + if len(avroFiles) != 1 { + return nil, fmt.Errorf("unexpected number of Avro files: expected 1, got %d", len(avroFiles)) + } + return avroFiles[0], nil } func (p *peerDBOCFWriter) WriteRecordsToAvroFile(ctx context.Context, filePath string) (*AvroFile, error) { @@ -261,14 +288,14 @@ func (p *peerDBOCFWriter) WriteRecordsToAvroFile(ctx context.Context, filePath s bufferedWriter := bufio.NewWriterSize(file, buffSizeBytes) defer bufferedWriter.Flush() - numRecords, err := p.WriteOCF(ctx, bufferedWriter) + numRecords, err := p.WriteOCF(ctx, bufferedWriter, -1) if err != nil { return nil, fmt.Errorf("failed to write records to temporary Avro file: %w", err) } printFileStats("finished writing to temporary Avro file") return &AvroFile{ - NumRecords: numRecords, + NumRecords: int(numRecords), StorageLocation: AvroLocalStorage, FilePath: filePath, }, nil diff --git a/flow/go.mod b/flow/go.mod index 54177874de..dcd4c361d5 100644 --- a/flow/go.mod +++ b/flow/go.mod @@ -5,35 +5,35 @@ go 1.23.0 require ( cloud.google.com/go v0.115.1 cloud.google.com/go/bigquery v1.62.0 - cloud.google.com/go/pubsub v1.42.0 + cloud.google.com/go/pubsub v1.43.0 cloud.google.com/go/storage v1.43.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 github.com/Azure/azure-sdk-for-go/sdk/messaging/azeventhubs v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub v1.2.0 - github.com/ClickHouse/clickhouse-go/v2 v2.28.2 + github.com/ClickHouse/clickhouse-go/v2 v2.28.3 github.com/PeerDB-io/glua64 v1.0.1 github.com/PeerDB-io/gluabit32 v1.0.2 github.com/PeerDB-io/gluaflatbuffers v1.0.1 github.com/PeerDB-io/gluajson v1.0.2 github.com/PeerDB-io/gluamsgpack v1.0.4 github.com/PeerDB-io/gluautf8 v1.0.0 - github.com/aws/aws-sdk-go-v2 v1.30.4 - github.com/aws/aws-sdk-go-v2/config v1.27.31 - github.com/aws/aws-sdk-go-v2/credentials v1.17.30 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.16 - github.com/aws/aws-sdk-go-v2/service/kms v1.35.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.61.0 - github.com/aws/aws-sdk-go-v2/service/ses v1.26.0 - github.com/aws/aws-sdk-go-v2/service/sns v1.31.5 + github.com/aws/aws-sdk-go-v2 v1.30.5 + github.com/aws/aws-sdk-go-v2/config v1.27.33 + github.com/aws/aws-sdk-go-v2/credentials v1.17.32 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18 + github.com/aws/aws-sdk-go-v2/service/kms v1.35.7 + github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 + github.com/aws/aws-sdk-go-v2/service/ses v1.26.2 + github.com/aws/aws-sdk-go-v2/service/sns v1.31.7 github.com/aws/smithy-go v1.20.4 github.com/cockroachdb/pebble v1.1.2 github.com/elastic/go-elasticsearch/v8 v8.15.0 github.com/google/uuid v1.6.0 - github.com/grafana/pyroscope-go v1.1.2 + github.com/grafana/pyroscope-go v1.2.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 github.com/jackc/pgerrcode v0.0.0-20240316143900-6e2875d9b438 github.com/jackc/pglogrepl v0.0.0-20240307033717-828fbfe908e9 - github.com/jackc/pgx/v5 v5.6.0 + github.com/jackc/pgx/v5 v5.7.1 github.com/jmoiron/sqlx v1.4.0 github.com/joho/godotenv v1.5.1 github.com/klauspost/compress v1.17.9 @@ -53,26 +53,26 @@ require ( github.com/urfave/cli/v3 v3.0.0-alpha9 github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 github.com/yuin/gopher-lua v1.1.1 - go.opentelemetry.io/otel v1.29.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.29.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.29.0 - go.opentelemetry.io/otel/metric v1.29.0 - go.opentelemetry.io/otel/sdk v1.29.0 - go.opentelemetry.io/otel/sdk/metric v1.29.0 + go.opentelemetry.io/otel v1.30.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 + go.opentelemetry.io/otel/metric v1.30.0 + go.opentelemetry.io/otel/sdk v1.30.0 + go.opentelemetry.io/otel/sdk/metric v1.30.0 go.temporal.io/api v1.39.0 - go.temporal.io/sdk v1.28.1 + go.temporal.io/sdk v1.29.1 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.26.0 - golang.org/x/mod v0.20.0 + golang.org/x/crypto v0.27.0 + golang.org/x/mod v0.21.0 golang.org/x/sync v0.8.0 - google.golang.org/api v0.195.0 + google.golang.org/api v0.197.0 google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 - google.golang.org/grpc v1.66.0 + google.golang.org/grpc v1.66.2 google.golang.org/protobuf v1.34.2 ) require ( - cloud.google.com/go/auth v0.9.2 // indirect + cloud.google.com/go/auth v0.9.4 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect @@ -80,11 +80,11 @@ require ( github.com/DataDog/zstd v1.5.6 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/apache/arrow/go/v15 v15.0.2 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect @@ -99,7 +99,7 @@ require ( github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect - github.com/getsentry/sentry-go v0.28.1 // indirect + github.com/getsentry/sentry-go v0.29.0 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.2 // indirect @@ -119,24 +119,24 @@ require ( github.com/nexus-rpc/sdk-go v0.0.10 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_golang v1.20.3 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.58.0 // indirect + github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/twmb/franz-go/pkg/kmsg v1.8.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect - golang.org/x/term v0.23.0 // indirect + golang.org/x/term v0.24.0 // indirect ) require ( - cloud.google.com/go/compute/metadata v0.5.0 // indirect - cloud.google.com/go/iam v1.2.0 // indirect + cloud.google.com/go/compute/metadata v0.5.1 // indirect + cloud.google.com/go/iam v1.2.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 // indirect @@ -144,13 +144,13 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/djherbis/buffer v1.2.0 github.com/djherbis/nio/v3 v3.0.1 @@ -165,14 +165,14 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v24.3.25+incompatible // indirect github.com/google/s2a-go v0.1.8 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.3 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kylelemons/godebug v1.1.0 // indirect @@ -185,13 +185,13 @@ require ( github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 - golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.24.0 // indirect + golang.org/x/tools v0.25.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/flow/go.sum b/flow/go.sum index ea536002c1..965076e5a6 100644 --- a/flow/go.sum +++ b/flow/go.sum @@ -1,24 +1,24 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= -cloud.google.com/go/auth v0.9.2 h1:I+Rq388FYU8QdbVB1IiPd+6KNdrqtAPE/asiKHShBLM= -cloud.google.com/go/auth v0.9.2/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk= +cloud.google.com/go/auth v0.9.4 h1:DxF7imbEbiFu9+zdKC6cKBko1e8XeJnipNqIbWZ+kDI= +cloud.google.com/go/auth v0.9.4/go.mod h1:SHia8n6//Ya940F1rLimhJCjjx7KE17t0ctFEci3HkA= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= cloud.google.com/go/bigquery v1.62.0 h1:SYEA2f7fKqbSRRBHb7g0iHTtZvtPSPYdXfmqsjpsBwo= cloud.google.com/go/bigquery v1.62.0/go.mod h1:5ee+ZkF1x/ntgCsFQJAQTM3QkAZOecfCmvxhkJsWRSA= -cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= -cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= +cloud.google.com/go/compute/metadata v0.5.1 h1:NM6oZeZNlYjiwYje+sYFjEpP0Q0zCan1bmQW/KmIrGs= +cloud.google.com/go/compute/metadata v0.5.1/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= cloud.google.com/go/datacatalog v1.22.0 h1:7e5/0B2LYbNx0BcUJbiCT8K2wCtcB5993z/v1JeLIdc= cloud.google.com/go/datacatalog v1.22.0/go.mod h1:4Wff6GphTY6guF5WphrD76jOdfBiflDiRGFAxq7t//I= -cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= -cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= +cloud.google.com/go/iam v1.2.1 h1:QFct02HRb7H12J/3utj0qf5tobFh9V4vR6h9eX5EBRU= +cloud.google.com/go/iam v1.2.1/go.mod h1:3VUIJDPpwT6p/amXRC5GY8fCCh70lxPygguVtI0Z4/g= cloud.google.com/go/kms v1.19.0 h1:x0OVJDl6UH1BSX4THKlMfdcFWoE4ruh90ZHuilZekrU= cloud.google.com/go/kms v1.19.0/go.mod h1:e4imokuPJUc17Trz2s6lEXFDt8bgDmvpVynH39bdrHM= cloud.google.com/go/longrunning v0.6.0 h1:mM1ZmaNsQsnb+5n1DNPeL0KwQd9jQRqSqSDEkBZr+aI= cloud.google.com/go/longrunning v0.6.0/go.mod h1:uHzSZqW89h7/pasCWNYdUpwGz3PcVWhrWupreVPYLts= -cloud.google.com/go/pubsub v1.42.0 h1:PVTbzorLryFL5ue8esTS2BfehUs0ahyNOY9qcd+HMOs= -cloud.google.com/go/pubsub v1.42.0/go.mod h1:KADJ6s4MbTwhXmse/50SebEhE4SmUwHi48z3/dHar1Y= +cloud.google.com/go/pubsub v1.43.0 h1:s3Qx+F96J7Kwey/uVHdK3QxFLIlOvvw4SfMYw2jFjb4= +cloud.google.com/go/pubsub v1.43.0/go.mod h1:LNLfqItblovg7mHWgU5g84Vhza4J8kTxx0YqIeTzcXY= cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= @@ -56,8 +56,8 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/ch-go v0.62.0 h1:eXH0hytXeCEEZHgMvOX9IiW7wqBb4w1MJMp9rArbkrc= github.com/ClickHouse/ch-go v0.62.0/go.mod h1:uzso52/PD9+gZj7tL6XAo8/EYDrx7CIwNF4c6PnO6S0= -github.com/ClickHouse/clickhouse-go/v2 v2.28.2 h1:D/sPEJzPRptJg6aaeAmm/ByDN9H9WgMGrgEl26QH1k8= -github.com/ClickHouse/clickhouse-go/v2 v2.28.2/go.mod h1:PQfZvFzU7TYkY68eCjc8Jq8M3HXC4hMnUmO0ZtVGkaM= +github.com/ClickHouse/clickhouse-go/v2 v2.28.3 h1:SkFzPULX6nzgfNZd1YD1XTECivjTMrCtD09ZPKcVLFQ= +github.com/ClickHouse/clickhouse-go/v2 v2.28.3/go.mod h1:vzn73hp+3JwxtFU4RjPCQ7r6fP2pMKVwdi8E1/Tkua8= github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= @@ -82,48 +82,48 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcyOsMLE= github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= -github.com/aws/aws-sdk-go-v2 v1.30.4 h1:frhcagrVNrzmT95RJImMHgabt99vkXGslubDaDagTk8= -github.com/aws/aws-sdk-go-v2 v1.30.4/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 h1:70PVAiL15/aBMh5LThwgXdSQorVr91L127ttckI9QQU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4/go.mod h1:/MQxMqci8tlqDH+pjmoLu1i0tbWCUP1hhyMRuFxpQCw= -github.com/aws/aws-sdk-go-v2/config v1.27.31 h1:kxBoRsjhT3pq0cKthgj6RU6bXTm/2SgdoUMyrVw0rAI= -github.com/aws/aws-sdk-go-v2/config v1.27.31/go.mod h1:z04nZdSWFPaDwK3DdJOG2r+scLQzMYuJeW0CujEm9FM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.30 h1:aau/oYFtibVovr2rDt8FHlU17BTicFEMAi29V1U+L5Q= -github.com/aws/aws-sdk-go-v2/credentials v1.17.30/go.mod h1:BPJ/yXV92ZVq6G8uYvbU0gSl8q94UB63nMT5ctNO38g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12 h1:yjwoSyDZF8Jth+mUk5lSPJCkMC0lMy6FaCD51jm6ayE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.12/go.mod h1:fuR57fAgMk7ot3WcNQfb6rSEn+SUffl7ri+aa8uKysI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.16 h1:1FWqcOnvnO0lRsv0kLACwwQquoZIoS5tD0MtfoNdnkk= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.16/go.mod h1:+E8OuB446P/5Swajo40TqenLMzm6aYDEEz6FZDn/u1E= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16 h1:TNyt/+X43KJ9IJJMjKfa3bNTiZbUP7DeCxfbTROESwY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.16/go.mod h1:2DwJF39FlNAUiX5pAc0UNeiz16lK2t7IaFcm0LFHEgc= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16 h1:jYfy8UPmd+6kJW5YhY0L1/KftReOGxI/4NtVSTh9O/I= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.16/go.mod h1:7ZfEPZxkW42Afq4uQB8H2E2e6ebh6mXTueEpYzjCzcs= +github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU= +github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18 h1:9DIp7vhmOPmueCDwpXa45bEbLHHTt1kcxChdTJWWxvI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.18/go.mod h1:aJv/Fwz8r56ozwYFRC4bzoeL1L17GYQYemfblOBux1M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16 h1:mimdLQkIX1zr8GIPY1ZtALdBQGxcASiBd2MOp8m/dMc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.16/go.mod h1:YHk6owoSwrIsok+cAH9PENCOGoH5PU2EllX4vLtSrsY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 h1:Roo69qTpfu8OlJ2Tb7pAYVuF0CpuUMB0IYWwYP/4DZM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17/go.mod h1:NcWPxQzGM1USQggaTVwz6VpqMZPX1CvDJLDh6jnOCa4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18 h1:GckUnpm4EJOAio1c8o25a+b3lVfwVzC9gnSBqiiNmZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.18/go.mod h1:Br6+bxfG33Dk3ynmkhsW2Z/t9D4+lRqdLDNCKi85w0U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18 h1:tJ5RnkHCiSH0jyd6gROjlJtNwov0eGYNz8s8nFcR0jQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.18/go.mod h1:++NHzT+nAF7ZPrHPsA+ENvsXkOO8wEu+C6RXltAG4/c= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16 h1:jg16PhLPUiHIj8zYIW6bqzeQSuHVEiWnGA0Brz5Xv2I= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.16/go.mod h1:Uyk1zE1VVdsHSU7096h/rwnXDzOzYQVl+FNPhPw7ShY= -github.com/aws/aws-sdk-go-v2/service/kms v1.35.5 h1:XUomV7SiclZl1QuXORdGcfFqHxEHET7rmNGtxTfNB+M= -github.com/aws/aws-sdk-go-v2/service/kms v1.35.5/go.mod h1:A5CS0VRmxxj2YKYLCY08l/Zzbd01m6JZn0WzxgT1OCA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.61.0 h1:Wb544Wh+xfSXqJ/j3R4aX9wrKUoZsJNmilBYZb3mKQ4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.61.0/go.mod h1:BSPI0EfnYUuNHPS0uqIo5VrRwzie+Fp+YhQOUs16sKI= -github.com/aws/aws-sdk-go-v2/service/ses v1.26.0 h1:k42jq8i0DbnPxr+URD58oZw/Esb93kzoSczfEYrPlw4= -github.com/aws/aws-sdk-go-v2/service/ses v1.26.0/go.mod h1:6Ul/Ir8oOCsI3dFN0prULK9fvpxP+WTYmlHDkFzaAVA= -github.com/aws/aws-sdk-go-v2/service/sns v1.31.5 h1:q8R1hxwOHE4e6TInafToa8AHTLQpJrxWXYk7GINJoyw= -github.com/aws/aws-sdk-go-v2/service/sns v1.31.5/go.mod h1:wDacBq+NshhM8KhdysbM4wRFxVyghyj7AAI+l8+o9f0= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.5 h1:zCsFCKvbj25i7p1u94imVoO447I/sFv8qq+lGJhRN0c= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.5/go.mod h1:ZeDX1SnKsVlejeuz41GiajjZpRSWR7/42q/EyA/QEiM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5 h1:SKvPgvdvmiTWoi0GAJ7AsJfOz3ngVkD/ERbs5pUnHNI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.5/go.mod h1:20sz31hv/WsPa3HhU3hfrIet2kxM4Pe0r20eBZ20Tac= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.5 h1:OMsEmCyz2i89XwRwPouAJvhj81wINh+4UK+k/0Yo/q8= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.5/go.mod h1:vmSqFK+BVIwVpDAGZB3CoCXHzurt4qBE8lf+I/kRTh0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 h1:FLMkfEiRjhgeDTCjjLoc3URo/TBkgeQbocA78lfkzSI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19/go.mod h1:Vx+GucNSsdhaxs3aZIKfSUjKVGsxN25nX2SRcdhuw08= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 h1:u+EfGmksnJc/x5tq3A+OD7LrMbSSR/5TrKLvkdy/fhY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17/go.mod h1:VaMx6302JHax2vHJWgRo+5n9zvbacs3bLU/23DNQrTY= +github.com/aws/aws-sdk-go-v2/service/kms v1.35.7 h1:v0D1LeMkA/X+JHAZWERrr+sUGOt8KrCZKnJA6KszkcE= +github.com/aws/aws-sdk-go-v2/service/kms v1.35.7/go.mod h1:K9lwD0Rsx9+NSaJKsdAdlDK4b2G4KKOEve9PzHxPoMI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2 h1:Kp6PWAlXwP1UvIflkIP6MFZYBNDCa4mFCGtxrpICVOg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2/go.mod h1:5FmD/Dqq57gP+XwaUnd5WFPipAuzrf0HmupX27Gvjvc= +github.com/aws/aws-sdk-go-v2/service/ses v1.26.2 h1:hGWgo0Ckz68QbnzET2ZlirsgIhSwa055Nlhbosr2944= +github.com/aws/aws-sdk-go-v2/service/ses v1.26.2/go.mod h1:PS2N1JNb+LsgIQA7Iu8PyoXWOUC5HAbt3esVYzWdVEg= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.7 h1:3MWDVQ1pS3e/S4ADKg+mMETqIbOuQDY9FqH7XCb5ISA= +github.com/aws/aws-sdk-go-v2/service/sns v1.31.7/go.mod h1:wjhxA9hlVu75dCL/5Wcx8Cwmszvu6t0i8WEDypcB4+s= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -181,8 +181,8 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= -github.com/getsentry/sentry-go v0.28.1 h1:zzaSm/vHmGllRM6Tpx1492r0YDzauArdBfkJRtY6P5k= -github.com/getsentry/sentry-go v0.28.1/go.mod h1:1fQZ+7l7eeJ3wYi82q5Hg8GqAPgefRq+FP/QhafYVgg= +github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA= +github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+jtBrlDEVWwLY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= @@ -257,15 +257,15 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.3 h1:QRje2j5GZimBzlbhGA2V2QlGNgL8G6e+wGo/+/2bWI0= -github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grafana/pyroscope-go v1.1.2 h1:7vCfdORYQMCxIzI3NlYAs3FcBP760+gWuYWOyiVyYx8= -github.com/grafana/pyroscope-go v1.1.2/go.mod h1:HSSmHo2KRn6FasBA4vK7BMiQqyQq8KSuBKvrhkXxYPU= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= @@ -286,10 +286,10 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -365,13 +365,13 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= -github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= +github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.58.0 h1:N+N8vY4/23r6iYfD3UQZUoJPnUYAo7v6LG5XZxjZTXo= -github.com/prometheus/common v0.58.0/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= +github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= +github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= @@ -437,35 +437,35 @@ github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI= -go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= +go.einride.tech/aip v0.68.0 h1:4seM66oLzTpz50u4K1zlJyOXQ3tCzcJN7I22tKkjipw= +go.einride.tech/aip v0.68.0/go.mod h1:7y9FF8VtPWqpxuAxl0KQWqaULxW4zFIesD6zF5RIHHg= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.29.0 h1:k6fQVDQexDE+3jG2SfCQjnHS7OamcP73YMoxEVq5B6k= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.29.0/go.mod h1:t4BrYLHU450Zo9fnydWlIuswB1bm7rM8havDpWOJeDo= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.29.0 h1:xvhQxJ/C9+RTnAj5DpTg7LSM1vbbMTiXt7e9hsfqHNw= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.29.0/go.mod h1:Fcvs2Bz1jkDM+Wf5/ozBGmi3tQ/c9zPKLnsipnfhGAo= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo= -go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok= -go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY= -go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 h1:hCq2hNMwsegUvPzI7sPOvtO9cqyy5GbWt/Ybp2xrx8Q= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0/go.mod h1:LqaApwGx/oUmzsbqxkzuBvyoPpkxk3JQWnqfVrJ3wCA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 h1:WypxHH02KX2poqqbaadmkMYalGyy/vil4HE4PM4nRJc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0/go.mod h1:U79SV99vtvGSEBeeHnpgGJfTsnsdkWLpPN/CcHAzBSI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= +go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.temporal.io/api v1.39.0 h1:pbhcfvNDB7mllb8lIBqPcg+m6LMG/IhTpdiFxe+0mYk= go.temporal.io/api v1.39.0/go.mod h1:1WwYUMo6lao8yl0371xWUm13paHExN5ATYT/B7QtFis= -go.temporal.io/sdk v1.28.1 h1:PsexsNDWXyWdJp4KWTOD+DfSZD1z0k5U/dIJF05akT4= -go.temporal.io/sdk v1.28.1/go.mod h1:zHcmZNXPaKXQJ6Hn98Ebcii7VlHL1mI4RJW8R6GQa1k= +go.temporal.io/sdk v1.29.1 h1:y+sUMbUhTU9rj50mwIZAPmcXCtgUdOWS9xHDYRYSgZ0= +go.temporal.io/sdk v1.29.1/go.mod h1:kp//DRvn3CqQVBCtjL51Oicp9wrZYB2s6row1UgzcKQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= @@ -476,11 +476,11 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= -golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -488,8 +488,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -501,11 +501,11 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -528,17 +528,17 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -551,8 +551,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -561,8 +561,8 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/api v0.195.0 h1:Ude4N8FvTKnnQJHU48RFI40jOBgIrL8Zqr3/QeST6yU= -google.golang.org/api v0.195.0/go.mod h1:DOGRWuv3P8TU8Lnz7uQc4hyNqrBpMtD9ppW3wBJurgc= +google.golang.org/api v0.197.0 h1:x6CwqQLsFiA5JKAiGyGBjc2bNtHtLddhJCE2IKuhhcQ= +google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -581,8 +581,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/flow/otel_metrics/peerdb_gauges/attributes.go b/flow/otel_metrics/peerdb_gauges/attributes.go index 70296a51d5..78b54b6119 100644 --- a/flow/otel_metrics/peerdb_gauges/attributes.go +++ b/flow/otel_metrics/peerdb_gauges/attributes.go @@ -3,5 +3,6 @@ package peerdb_gauges const ( PeerNameKey string = "peerName" SlotNameKey string = "slotName" + FlowNameKey string = "flowName" DeploymentUidKey string = "deploymentUID" ) diff --git a/flow/peerdbenv/dynamicconf.go b/flow/peerdbenv/dynamicconf.go index bb1a6011df..c126352b81 100644 --- a/flow/peerdbenv/dynamicconf.go +++ b/flow/peerdbenv/dynamicconf.go @@ -124,6 +124,12 @@ END;`, ValueType: protos.DynconfValueType_STRING, ApplyMode: protos.DynconfApplyMode_APPLY_MODE_NEW_MIRROR, TargetForSetting: protos.DynconfTarget_BIGQUERY, }, + { + Name: "PEERDB_CLICKHOUSE_NUM_ROWS_PER_AVRO_FILE", DefaultValue: "1000000", ValueType: protos.DynconfValueType_UINT, + Description: "Number of rows per Avro file for ClickHouse mirrors", + ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE, + TargetForSetting: protos.DynconfTarget_CLICKHOUSE, + }, } var DynamicIndex = func() map[string]int { @@ -290,3 +296,7 @@ func PeerDBQueueForceTopicCreation(ctx context.Context, env map[string]string) ( func PeerDBMaxSyncsPerCDCFlow(ctx context.Context, env map[string]string) (uint32, error) { return dynamicConfUnsigned[uint32](ctx, env, "PEERDB_MAX_SYNCS_PER_CDC_FLOW") } + +func PeerDBClickhouseNumRowsPerAvroFile(ctx context.Context, env map[string]string) (uint32, error) { + return dynamicConfUnsigned[uint32](ctx, env, "PEERDB_CLICKHOUSE_NUM_ROWS_PER_AVRO_FILE") +} diff --git a/nexus/Cargo.lock b/nexus/Cargo.lock index 5e9df5a343..11d28fdc5b 100644 --- a/nexus/Cargo.lock +++ b/nexus/Cargo.lock @@ -4,19 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -162,9 +156,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "ar" @@ -255,9 +249,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-config" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e95816a168520d72c0e7680c405a5a8c1fb6a035b4bc4b9d7b0de8e1a941697" +checksum = "848d7b9b605720989929279fa644ce8f244d0ce3146fcca5b70e4eb7b3c020fc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -297,9 +291,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2424565416eef55906f9f8cece2072b6b6a76075e3ff81483ebe938a89a4c05f" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -322,9 +316,9 @@ dependencies = [ [[package]] name = "aws-sdk-kms" -version = "1.41.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "178910fefe72743b62b9c4670c14a038ebfdb265ff7feccf43827af6a8899e14" +checksum = "d9f7cb482caa5444d445c94417b9c74e49a849beb09ede4f2f4c3c15f8157387" dependencies = [ "aws-credential-types", "aws-runtime", @@ -344,9 +338,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.40.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5879bec6e74b648ce12f6085e7245417bc5f6d672781028384d2e494be3eb6d" +checksum = "27bf24cd0d389daa923e974b0e7c38daf308fc21e963c049f57980235017175e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -366,9 +360,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.41.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ef4cd9362f638c22a3b959fd8df292e7e47fdf170270f86246b97109b5f2f7d" +checksum = "3b43b3220f1c46ac0e9dcc0a97d94b93305dacb36d1dd393996300c6b9b74364" dependencies = [ "aws-credential-types", "aws-runtime", @@ -388,9 +382,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.40.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1e2735d2ab28b35ecbb5496c9d41857f52a0d6a0075bbf6a8af306045ea6f6" +checksum = "d1c46924fb1add65bba55636e12812cae2febf68c0f37361766f627ddcca91ce" dependencies = [ "aws-credential-types", "aws-runtime", @@ -411,9 +405,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -445,9 +439,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.10" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01dbcb6e2588fd64cfb6d7529661b06466419e4c54ed1c62d6510d2d0350a728" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -528,9 +522,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.4" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273dcdfd762fae3e1650b8024624e7cd50e484e37abdab73a7a706188ad34543" +checksum = "03701449087215b5369c7ea17fef0dd5d24cb93439ec5af0c7615f58c3f22605" dependencies = [ "base64-simd", "bytes", @@ -554,9 +548,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.8" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] @@ -624,17 +618,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -883,9 +877,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.15" +version = "1.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6" +checksum = "2d74707dde2ba56f86ae90effb3b43ddd369504387e718014de010cec7959800" dependencies = [ "jobserver", "libc", @@ -976,9 +970,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -986,9 +980,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstream", "anstyle", @@ -1053,9 +1047,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1137,9 +1131,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1335,7 +1329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -1522,9 +1516,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -1753,16 +1747,16 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.12", - "rustls-native-certs 0.7.3", + "rustls 0.23.13", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls 0.26.0", @@ -1785,9 +1779,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -1868,9 +1862,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "is_terminal_polyfill" @@ -2074,15 +2068,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", -] - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2307,9 +2292,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe" [[package]] name = "opaque-debug" @@ -2773,9 +2758,9 @@ dependencies = [ [[package]] name = "postgres" -version = "0.19.8" +version = "0.19.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c9ec84ab55b0f9e418675de50052d494ba893fd28c65769a6e68fcdacbee2b8" +checksum = "95c918733159f4d55d2ceb262950f00b0aebd6af4aa97b5a47bb0655120475ed" dependencies = [ "bytes", "fallible-iterator", @@ -2791,7 +2776,7 @@ version = "0.1.0" dependencies = [ "anyhow", "pt", - "rustls 0.23.12", + "rustls 0.23.13", "tokio", "tokio-postgres", "tokio-postgres-rustls", @@ -2829,9 +2814,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02048d9e032fb3cc3413bbf7b83a15d84a5d419778e2628751896d856498eee9" +checksum = "f66ea23a2d0e5734297357705193335e0a957696f34bed2f2faefacb2fec336f" dependencies = [ "array-init", "bytes", @@ -3037,7 +3022,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.13", "socket2", "thiserror", "tokio", @@ -3054,7 +3039,7 @@ dependencies = [ "rand", "ring", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.13", "slab", "thiserror", "tinyvec", @@ -3141,9 +3126,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags", ] @@ -3265,7 +3250,7 @@ dependencies = [ "http-body 1.0.1", "http-body-util", "hyper 1.4.1", - "hyper-rustls 0.27.2", + "hyper-rustls 0.27.3", "hyper-util", "ipnet", "js-sys", @@ -3275,7 +3260,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", @@ -3404,9 +3389,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.35" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a85d50532239da68e9addb745ba38ff4612a242c1c7ceea689c4bc7c2f43c36f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags", "errno", @@ -3436,22 +3421,22 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.7", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -3481,6 +3466,19 @@ dependencies = [ "security-framework", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +dependencies = [ + "openssl-probe", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -3518,9 +3516,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -3565,11 +3563,11 @@ checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3645,9 +3643,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] @@ -3663,9 +3661,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", @@ -3674,9 +3672,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -4089,9 +4087,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03adcf0147e203b6032c0b2d30be1415ba03bc348901f3ff1cc0df6a733e60c3" +checksum = "3b5d3742945bc7d7f210693b0c58ae542c6fd47b17adbbda0885f3dcb34a6bdb" dependencies = [ "async-trait", "byteorder", @@ -4120,7 +4118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab" dependencies = [ "ring", - "rustls 0.23.12", + "rustls 0.23.13", "tokio", "tokio-postgres", "tokio-rustls 0.26.0", @@ -4154,16 +4152,16 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -4172,9 +4170,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -4434,9 +4432,9 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" @@ -4486,7 +4484,7 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pki-types", "serde", "serde_json", @@ -4984,11 +4982,11 @@ dependencies = [ "http 1.1.0", "http-body-util", "hyper 1.4.1", - "hyper-rustls 0.27.2", + "hyper-rustls 0.27.3", "hyper-util", "log", "percent-encoding", - "rustls 0.23.12", + "rustls 0.23.13", "rustls-pemfile 2.1.3", "seahash", "serde", diff --git a/nexus/catalog/migrations/V38__alerting_config_mirror_names.sql b/nexus/catalog/migrations/V38__alerting_config_mirror_names.sql new file mode 100644 index 0000000000..ea878a3fc3 --- /dev/null +++ b/nexus/catalog/migrations/V38__alerting_config_mirror_names.sql @@ -0,0 +1 @@ +ALTER TABLE peerdb_stats.alerting_config ADD COLUMN IF NOT EXISTS alert_for_mirrors TEXT[]; diff --git a/nexus/peer-bigquery/src/ast.rs b/nexus/peer-bigquery/src/ast.rs index fd8153382a..865a69cefe 100644 --- a/nexus/peer-bigquery/src/ast.rs +++ b/nexus/peer-bigquery/src/ast.rs @@ -13,14 +13,10 @@ pub struct BigqueryAst; impl BigqueryAst { pub fn is_timestamp_returning_function(&self, name: &str) -> bool { - if name.eq_ignore_ascii_case("now") + name.eq_ignore_ascii_case("now") || name.eq_ignore_ascii_case("date_trunc") || name.eq_ignore_ascii_case("make_timestamp") || name.eq_ignore_ascii_case("current_timestamp") - { - return true; - } - false } pub fn is_timestamp_expr(&self, e: &Expr) -> bool { @@ -68,10 +64,12 @@ impl BigqueryAst { None } - pub fn rewrite(&self, dataset: &str, query: &mut Query) -> anyhow::Result<()> { + pub fn rewrite(&self, peername: &str, dataset: &str, query: &mut Query) -> anyhow::Result<()> { // replace peername with the connected dataset. visit_relations_mut(query, |table| { - table.0[0] = dataset.into(); + if table.0.len() > 1 && peername.eq_ignore_ascii_case(&table.0[0].value) { + table.0[0] = dataset.into(); + } ControlFlow::<()>::Continue(()) }); diff --git a/nexus/peer-bigquery/src/lib.rs b/nexus/peer-bigquery/src/lib.rs index e251081aa1..9cdb01c0cb 100644 --- a/nexus/peer-bigquery/src/lib.rs +++ b/nexus/peer-bigquery/src/lib.rs @@ -98,7 +98,7 @@ impl QueryExecutor for BigQueryQueryExecutor { Statement::Query(query) => { let mut query = query.clone(); ast::BigqueryAst - .rewrite(&self.dataset_id, &mut query) + .rewrite(&self.peer_name, &self.dataset_id, &mut query) .context("unable to rewrite query") .map_err(|err| PgWireError::ApiError(err.into()))?; @@ -210,7 +210,7 @@ impl QueryExecutor for BigQueryQueryExecutor { Statement::Query(query) => { let mut query = query.clone(); ast::BigqueryAst - .rewrite(&self.dataset_id, &mut query) + .rewrite(&self.peer_name, &self.dataset_id, &mut query) .context("unable to rewrite query") .map_err(|err| PgWireError::ApiError(err.into()))?; diff --git a/nexus/peer-postgres/src/ast.rs b/nexus/peer-postgres/src/ast.rs index 9e76b9eb76..ad21dbaf57 100644 --- a/nexus/peer-postgres/src/ast.rs +++ b/nexus/peer-postgres/src/ast.rs @@ -12,7 +12,7 @@ impl PostgresAst { visit_relations_mut(query, |table| { // if peer name is first part of table name, remove first part if let Some(ref peername) = self.peername { - if peername.eq_ignore_ascii_case(&table.0[0].value) { + if table.0.len() > 1 && peername.eq_ignore_ascii_case(&table.0[0].value) { table.0.remove(0); } } diff --git a/nexus/peer-snowflake/src/ast.rs b/nexus/peer-snowflake/src/ast.rs index 0934ec5592..ad6ab6a882 100644 --- a/nexus/peer-snowflake/src/ast.rs +++ b/nexus/peer-snowflake/src/ast.rs @@ -10,7 +10,9 @@ pub struct SnowflakeAst; impl SnowflakeAst { pub fn rewrite(&self, query: &mut Query) -> anyhow::Result<()> { visit_relations_mut(query, |table| { - table.0.remove(0); + if table.0.len() > 1 { + table.0.remove(0); + } ControlFlow::<()>::Continue(()) }); diff --git a/protos/route.proto b/protos/route.proto index ad755ebcd3..fb30bbb423 100644 --- a/protos/route.proto +++ b/protos/route.proto @@ -41,13 +41,13 @@ message AlertConfig { int32 id = 1; string service_type = 2; string service_config = 3; + repeated string alert_for_mirrors = 4; } message GetAlertConfigsRequest { } + message PostAlertConfigRequest { - int32 id = 1; - string service_type = 2; - string service_config = 3; + AlertConfig config = 1; } message DeleteAlertConfigRequest { int32 id = 1; diff --git a/ui/app/alert-config/new.tsx b/ui/app/alert-config/new.tsx index b188c1862a..f399b7af16 100644 --- a/ui/app/alert-config/new.tsx +++ b/ui/app/alert-config/new.tsx @@ -1,3 +1,4 @@ +import { PostAlertConfigRequest } from '@/grpc_generated/route'; import { Button } from '@/lib/Button'; import { Label } from '@/lib/Label/Label'; import { TextField } from '@/lib/TextField'; @@ -26,6 +27,7 @@ export interface AlertConfigProps { serviceType: ServiceType; alertConfig: serviceConfigType; forEdit?: boolean; + alertForMirrors?: string[]; } function ConfigLabel(data: { label: string; value: string }) { @@ -161,6 +163,10 @@ export function NewConfig(alertProps: AlertConfigProps) { alertProps.alertConfig ); + const [alertForMirrors, setAlertForMirrors] = useState( + alertProps.alertForMirrors || [] + ); + const [loading, setLoading] = useState(false); const handleAdd = async () => { @@ -185,6 +191,7 @@ export function NewConfig(alertProps: AlertConfigProps) { id: Number(alertProps.id || -1), serviceType, serviceConfig, + alertForMirrors, }; const alertReqValidity = alertConfigReqSchema.safeParse(alertConfigReq); @@ -192,14 +199,23 @@ export function NewConfig(alertProps: AlertConfigProps) { notifyErr(alertReqValidity.error.issues[0].message); return; } - (alertConfigReq as any).serviceConfig = JSON.stringify( - alertConfigReq.serviceConfig - ); + + const alertConfigProtoReq: PostAlertConfigRequest = { + config: { + id: alertConfigReq.id || -1, + serviceType: alertConfigReq.serviceType, + serviceConfig: JSON.stringify(alertConfigReq.serviceConfig), + alertForMirrors: + alertConfigReq.alertForMirrors?.filter( + (mirror) => mirror && mirror.trim() !== '' + ) || [], + }, + }; setLoading(true); const createRes = await fetch('/api/v1/alerts/config', { method: 'POST', - body: JSON.stringify(alertConfigReq), + body: JSON.stringify(alertConfigProtoReq), }); setLoading(false); @@ -289,6 +305,19 @@ export function NewConfig(alertProps: AlertConfigProps) { } /> +
+

+ Alert only for these mirrors (leave empty to alert for all mirrors) +

+ setAlertForMirrors(e.target.value.split(','))} + /> +
{ServiceFields} + ); +}; + +export default ProjectCard; diff --git a/ui/app/supabase/styles.tsx b/ui/app/supabase/styles.tsx new file mode 100644 index 0000000000..aa9b52517f --- /dev/null +++ b/ui/app/supabase/styles.tsx @@ -0,0 +1,42 @@ +export const ProjectsContainerStyle: React.CSSProperties = { + width: '100%', + height: '100%', + display: 'flex', + flexDirection: 'column', + alignItems: 'center', + justifyContent: 'center', + rowGap: '1rem', +}; + +export const ProjectListStyle: React.CSSProperties = { + display: 'flex', + flexDirection: 'column', + rowGap: '1rem', + maxHeight: '40%', + overflowY: 'auto', + width: '30%', +}; + +export const ProjectCardStyle: React.CSSProperties = { + display: 'flex', + flexDirection: 'column', + alignItems: 'flex-start', + fontSize: 13, + padding: '0.5rem', + borderRadius: '1rem', + backgroundColor: 'white', + boxShadow: '0px 0px 2px rgba(0, 0, 0, 0.1)', + border: '1px solid rgba(0, 0, 0, 0.1)', + cursor: 'pointer', + width: '100%', +}; + +export const ProjectNameStyle: React.CSSProperties = { + width: '70%', + padding: '4px 8px', + alignItems: 'start', + textAlign: 'left', + overflowX: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', +}; diff --git a/ui/components/PeerComponent.tsx b/ui/components/PeerComponent.tsx index 6454575e84..aeef4926cf 100644 --- a/ui/components/PeerComponent.tsx +++ b/ui/components/PeerComponent.tsx @@ -20,6 +20,8 @@ export const DBTypeToImageMapping = (peerType: DBType | string) => { return '/images/crunchy.png'; case 'NEON': return '/images/neon.png'; + case 'SUPABASE': + return '/svgs/supabase-logo-icon.svg'; case DBType.POSTGRES: case 'POSTGRES': return '/svgs/pg.svg'; @@ -57,13 +59,13 @@ export const DBTypeToImageMapping = (peerType: DBType | string) => { } }; -const PeerButton = ({ +export default function PeerButton({ peerName, peerType, }: { peerName: string; peerType: DBType; -}) => { +}) { const router = useRouter(); const [isLoading, setIsLoading] = useState(false); @@ -94,6 +96,4 @@ const PeerButton = ({ ); -}; - -export default PeerButton; +} diff --git a/ui/components/PeerForms/PostgresForm.tsx b/ui/components/PeerForms/PostgresForm.tsx index 1f69ff90a5..d030c2b89b 100644 --- a/ui/components/PeerForms/PostgresForm.tsx +++ b/ui/components/PeerForms/PostgresForm.tsx @@ -1,5 +1,5 @@ 'use client'; -import { PeerSetter } from '@/app/dto/PeersDTO'; +import { PeerConfig, PeerSetter } from '@/app/dto/PeersDTO'; import { PeerSetting } from '@/app/peers/create/[peerType]/helpers/common'; import { SSHSetting, @@ -13,15 +13,23 @@ import { RowWithTextField } from '@/lib/Layout'; import { Switch } from '@/lib/Switch'; import { TextField } from '@/lib/TextField'; import { Tooltip } from '@/lib/Tooltip'; +import { useSearchParams } from 'next/navigation'; import { useEffect, useState } from 'react'; import { InfoPopover } from '../InfoPopover'; interface ConfigProps { settings: PeerSetting[]; setter: PeerSetter; + config: PeerConfig; type: string; } -export default function PostgresForm({ settings, setter, type }: ConfigProps) { +export default function PostgresForm({ + settings, + config, + setter, + type, +}: ConfigProps) { + const searchParams = useSearchParams(); const [showSSH, setShowSSH] = useState(false); const [sshConfig, setSSHConfig] = useState(blankSSHConfig); const handleFile = ( @@ -45,13 +53,6 @@ export default function PostgresForm({ settings, setter, type }: ConfigProps) { } }; - const handleChange = ( - e: React.ChangeEvent, - setting: PeerSetting - ) => { - setting.stateHandler(e.target.value, setter); - }; - const handleSSHParam = ( e: React.ChangeEvent, setting: SSHSetting @@ -64,12 +65,17 @@ export default function PostgresForm({ settings, setter, type }: ConfigProps) { }; useEffect(() => { - setter((prev) => { - return { - ...prev, - sshConfig: showSSH ? sshConfig : undefined, - }; - }); + const host = searchParams.get('host'); + if (host) setter((curr) => ({ ...curr, host })); + const database = searchParams.get('db'); + if (database) setter((curr) => ({ ...curr, database })); + }, [setter, searchParams]); + + useEffect(() => { + setter((prev) => ({ + ...prev, + sshConfig: showSSH ? sshConfig : undefined, + })); }, [sshConfig, setter, showSSH]); return ( @@ -105,8 +111,11 @@ export default function PostgresForm({ settings, setter, type }: ConfigProps) { variant='simple' type={setting.type} defaultValue={setting.default} + value={ + setting.field && config[setting.field as keyof PeerConfig] + } onChange={(e: React.ChangeEvent) => - handleChange(e, setting) + setting.stateHandler(e.target.value, setter) } /> {setting.tips && ( diff --git a/ui/components/SelectSource.tsx b/ui/components/SelectSource.tsx index 951eabfd2f..7df186a543 100644 --- a/ui/components/SelectSource.tsx +++ b/ui/components/SelectSource.tsx @@ -9,12 +9,12 @@ import useSWR from 'swr'; import { DBTypeToImageMapping } from './PeerComponent'; // label corresponds to PeerType -function SourceLabel({ label }: { label: string }) { +function SourceLabel({ label, url }: { label: string; url?: string }) { const peerLogo = DBTypeToImageMapping(label); return (