+
+
+
#### **Postgres-compatible SQL interface to do ETL**
The Postgres-compatible SQL interface for ETL is unique to PeerDB and enables you to operate in a language you are familiar with. You can do ETL the same way you work with your databases.
diff --git a/docker-bake.hcl b/docker-bake.hcl
index 6e6098ca14..4927cd5505 100644
--- a/docker-bake.hcl
+++ b/docker-bake.hcl
@@ -16,6 +16,7 @@ group "default" {
"flow-worker",
"flow-api",
"flow-snapshot-worker",
+ "flow-maintenance",
"peerdb-ui"
]
}
@@ -45,6 +46,9 @@ target "flow-snapshot-worker" {
"linux/amd64",
"linux/arm64",
]
+ args = {
+ PEERDB_VERSION_SHA_SHORT = "${SHA_SHORT}"
+ }
tags = [
"${REGISTRY}/flow-snapshot-worker:${TAG}",
"${REGISTRY}/flow-snapshot-worker:${SHA_SHORT}",
@@ -59,12 +63,32 @@ target "flow-worker" {
"linux/amd64",
"linux/arm64",
]
+ args = {
+ PEERDB_VERSION_SHA_SHORT = "${SHA_SHORT}"
+ }
tags = [
"${REGISTRY}/flow-worker:${TAG}",
"${REGISTRY}/flow-worker:${SHA_SHORT}",
]
}
+target "flow-maintenance" {
+ context = "."
+ dockerfile = "stacks/flow.Dockerfile"
+ target = "flow-maintenance"
+ platforms = [
+ "linux/amd64",
+ "linux/arm64",
+ ]
+ args = {
+ PEERDB_VERSION_SHA_SHORT = "${SHA_SHORT}"
+ }
+ tags = [
+ "${REGISTRY}/flow-maintenance:${TAG}",
+ "${REGISTRY}/flow-maintenance:${SHA_SHORT}",
+ ]
+}
+
target "peerdb" {
context = "."
dockerfile = "stacks/peerdb-server.Dockerfile"
@@ -72,6 +96,9 @@ target "peerdb" {
"linux/amd64",
"linux/arm64",
]
+ args = {
+ PEERDB_VERSION_SHA_SHORT = "${SHA_SHORT}"
+ }
tags = [
"${REGISTRY}/peerdb-server:${TAG}",
"${REGISTRY}/peerdb-server:${SHA_SHORT}",
@@ -85,6 +112,9 @@ target "peerdb-ui" {
"linux/amd64",
"linux/arm64",
]
+ args = {
+ PEERDB_VERSION_SHA_SHORT = "${SHA_SHORT}"
+ }
tags = [
"${REGISTRY}/peerdb-ui:${TAG}",
"${REGISTRY}/peerdb-ui:${SHA_SHORT}",
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 7110819257..6459c0b131 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -39,7 +39,7 @@ x-flow-worker-env: &flow-worker-env
services:
catalog:
container_name: catalog
- image: postgres:16-alpine
+ image: postgres:17-alpine@sha256:e7897baa70dae1968d23d785adb4aeb699175e0bcaae44f98a7083ecb9668b93
command: -c config_file=/etc/postgresql.conf
ports:
- 9901:5432
@@ -73,7 +73,7 @@ services:
- POSTGRES_PWD=postgres
- POSTGRES_SEEDS=catalog
- DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
- image: temporalio/auto-setup:1.25
+ image: temporalio/auto-setup:1.25@sha256:b1edc1e20002d958c8182f2ae08dee877a125083683a627a44917683419ba6a8
ports:
- 7233:7233
volumes:
@@ -83,7 +83,7 @@ services:
pyroscope:
container_name: pyroscope
- image: grafana/pyroscope:latest
+ image: grafana/pyroscope:latest@sha256:319bf32ae06b67c1b9795c06ae6c3ba67e9b43382896df7a9df54cdb47a5c535
ports:
- 4040:4040
@@ -95,7 +95,7 @@ services:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CLI_ADDRESS=temporal:7233
- TEMPORAL_CLI_SHOW_STACKS=1
- image: temporalio/admin-tools:1.25.2-tctl-1.18.1-cli-1.1.1
+ image: temporalio/admin-tools:1.25.2-tctl-1.18.1-cli-1.1.1@sha256:da0c7a7982b571857173ab8f058e7f139b3054800abb4dcb100445d29a563ee8
stdin_open: true
tty: true
entrypoint: /etc/temporal/entrypoint.sh
@@ -116,7 +116,7 @@ services:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CORS_ORIGINS=http://localhost:3000
- TEMPORAL_CSRF_COOKIE_INSECURE=true
- image: temporalio/ui:2.29.1
+ image: temporalio/ui:2.32.0@sha256:82bf98dbe005a831b6bc5dc12ccd7bffd606af2032dae4821ae133caaa943d3d
ports:
- 8085:8080
@@ -209,7 +209,7 @@ services:
- flow-api
minio:
- image: minio/minio:RELEASE.2024-11-07T00-52-20Z
+ image: minio/minio:RELEASE.2024-11-07T00-52-20Z@sha256:ac591851803a79aee64bc37f66d77c56b0a4b6e12d9e5356380f4105510f2332
volumes:
- minio-data:/data
ports:
diff --git a/docker-compose.yml b/docker-compose.yml
index ce4a3994ad..b2c5936a2b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -32,7 +32,7 @@ x-flow-worker-env: &flow-worker-env
services:
catalog:
container_name: catalog
- image: postgres:16-alpine
+ image: postgres:17-alpine@sha256:e7897baa70dae1968d23d785adb4aeb699175e0bcaae44f98a7083ecb9668b93
command: -c config_file=/etc/postgresql.conf
restart: unless-stopped
ports:
@@ -68,7 +68,7 @@ services:
- POSTGRES_PWD=postgres
- POSTGRES_SEEDS=catalog
- DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development-sql.yaml
- image: temporalio/auto-setup:1.25
+ image: temporalio/auto-setup:1.25@sha256:b1edc1e20002d958c8182f2ae08dee877a125083683a627a44917683419ba6a8
ports:
- 7233:7233
volumes:
@@ -85,7 +85,7 @@ services:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CLI_ADDRESS=temporal:7233
- TEMPORAL_CLI_SHOW_STACKS=1
- image: temporalio/admin-tools:1.25.2-tctl-1.18.1-cli-1.1.1
+ image: temporalio/admin-tools:1.25.2-tctl-1.18.1-cli-1.1.1@sha256:da0c7a7982b571857173ab8f058e7f139b3054800abb4dcb100445d29a563ee8
stdin_open: true
tty: true
entrypoint: /etc/temporal/entrypoint.sh
@@ -106,13 +106,13 @@ services:
- TEMPORAL_ADDRESS=temporal:7233
- TEMPORAL_CORS_ORIGINS=http://localhost:3000
- TEMPORAL_CSRF_COOKIE_INSECURE=true
- image: temporalio/ui:2.29.1
+ image: temporalio/ui:2.32.0@sha256:82bf98dbe005a831b6bc5dc12ccd7bffd606af2032dae4821ae133caaa943d3d
ports:
- 8085:8080
flow-api:
container_name: flow_api
- image: ghcr.io/peerdb-io/flow-api:latest-dev
+ image: ghcr.io/peerdb-io/flow-api:stable-v0.20.0
restart: unless-stopped
ports:
- 8112:8112
@@ -128,7 +128,7 @@ services:
flow-snapshot-worker:
container_name: flow-snapshot-worker
- image: ghcr.io/peerdb-io/flow-snapshot-worker:latest-dev
+ image: ghcr.io/peerdb-io/flow-snapshot-worker:stable-v0.20.0
restart: unless-stopped
environment:
<<: [*catalog-config, *flow-worker-env, *minio-config]
@@ -138,7 +138,7 @@ services:
flow-worker:
container_name: flow-worker
- image: ghcr.io/peerdb-io/flow-worker:latest-dev
+ image: ghcr.io/peerdb-io/flow-worker:stable-v0.20.0
restart: unless-stopped
environment:
<<: [*catalog-config, *flow-worker-env, *minio-config]
@@ -151,7 +151,7 @@ services:
peerdb:
container_name: peerdb-server
stop_signal: SIGINT
- image: ghcr.io/peerdb-io/peerdb-server:latest-dev
+ image: ghcr.io/peerdb-io/peerdb-server:stable-v0.20.0
restart: unless-stopped
environment:
<<: *catalog-config
@@ -167,7 +167,7 @@ services:
peerdb-ui:
container_name: peerdb-ui
- image: ghcr.io/peerdb-io/peerdb-ui:latest-dev
+ image: ghcr.io/peerdb-io/peerdb-ui:stable-v0.20.0
restart: unless-stopped
ports:
- 3000:3000
@@ -184,7 +184,7 @@ services:
- flow-api
minio:
- image: minio/minio:RELEASE.2024-11-07T00-52-20Z
+ image: minio/minio:RELEASE.2024-11-07T00-52-20Z@sha256:ac591851803a79aee64bc37f66d77c56b0a4b6e12d9e5356380f4105510f2332
restart: unless-stopped
volumes:
- minio-data:/data
diff --git a/flow/activities/flowable.go b/flow/activities/flowable.go
index f6d785e434..bd7d0ee922 100644
--- a/flow/activities/flowable.go
+++ b/flow/activities/flowable.go
@@ -28,7 +28,6 @@ import (
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/otel_metrics"
- "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_gauges"
"github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/pua"
"github.com/PeerDB-io/peer-flow/shared"
@@ -293,11 +292,13 @@ func (a *FlowableActivity) MaintainPull(
ctx = context.WithValue(ctx, shared.FlowNameKey, config.FlowJobName)
srcConn, err := connectors.GetByNameAs[connectors.CDCPullConnector](ctx, config.Env, a.CatalogPool, config.SourceName)
if err != nil {
+ a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return err
}
defer connectors.CloseConnector(ctx, srcConn)
if err := srcConn.SetupReplConn(ctx); err != nil {
+ a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return err
}
@@ -413,7 +414,7 @@ func (a *FlowableActivity) StartNormalize(
if errors.Is(err, errors.ErrUnsupported) {
return nil, monitoring.UpdateEndTimeForCDCBatch(ctx, a.CatalogPool, input.FlowConnectionConfigs.FlowJobName, input.SyncBatchID)
} else if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to get normalize connector: %w", err)
}
defer connectors.CloseConnector(ctx, dstConn)
@@ -424,7 +425,7 @@ func (a *FlowableActivity) StartNormalize(
tableNameSchemaMapping, err := a.getTableNameSchemaMapping(ctx, input.FlowConnectionConfigs.FlowJobName)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to get table name schema mapping: %w", err)
}
res, err := dstConn.NormalizeRecords(ctx, &model.NormalizeRecordsRequest{
@@ -442,13 +443,13 @@ func (a *FlowableActivity) StartNormalize(
}
dstType, err := connectors.LoadPeerType(ctx, a.CatalogPool, input.FlowConnectionConfigs.DestinationName)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to get peer type: %w", err)
}
if dstType == protos.DBType_POSTGRES {
err = monitoring.UpdateEndTimeForCDCBatch(ctx, a.CatalogPool, input.FlowConnectionConfigs.FlowJobName,
input.SyncBatchID)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to update end time for cdc batch: %w", err)
}
}
@@ -770,11 +771,10 @@ func (a *FlowableActivity) RecordSlotSizes(ctx context.Context) error {
return
}
- slotMetricGauges := peerdb_gauges.SlotMetricGauges{}
+ slotMetricGauges := otel_metrics.SlotMetricGauges{}
if a.OtelManager != nil {
- slotLagGauge, err := otel_metrics.GetOrInitFloat64SyncGauge(a.OtelManager.Meter,
- a.OtelManager.Float64GaugesCache,
- peerdb_gauges.BuildGaugeName(peerdb_gauges.SlotLagGaugeName),
+ slotLagGauge, err := a.OtelManager.GetOrInitFloat64Gauge(
+ otel_metrics.BuildMetricName(otel_metrics.SlotLagGaugeName),
metric.WithUnit("MiBy"),
metric.WithDescription("Postgres replication slot lag in MB"))
if err != nil {
@@ -783,9 +783,8 @@ func (a *FlowableActivity) RecordSlotSizes(ctx context.Context) error {
}
slotMetricGauges.SlotLagGauge = slotLagGauge
- openConnectionsGauge, err := otel_metrics.GetOrInitInt64SyncGauge(a.OtelManager.Meter,
- a.OtelManager.Int64GaugesCache,
- peerdb_gauges.BuildGaugeName(peerdb_gauges.OpenConnectionsGaugeName),
+ openConnectionsGauge, err := a.OtelManager.GetOrInitInt64Gauge(
+ otel_metrics.BuildMetricName(otel_metrics.OpenConnectionsGaugeName),
metric.WithDescription("Current open connections for PeerDB user"))
if err != nil {
logger.Error("Failed to get open connections gauge", slog.Any("error", err))
@@ -793,9 +792,8 @@ func (a *FlowableActivity) RecordSlotSizes(ctx context.Context) error {
}
slotMetricGauges.OpenConnectionsGauge = openConnectionsGauge
- openReplicationConnectionsGauge, err := otel_metrics.GetOrInitInt64SyncGauge(a.OtelManager.Meter,
- a.OtelManager.Int64GaugesCache,
- peerdb_gauges.BuildGaugeName(peerdb_gauges.OpenReplicationConnectionsGaugeName),
+ openReplicationConnectionsGauge, err := a.OtelManager.GetOrInitInt64Gauge(
+ otel_metrics.BuildMetricName(otel_metrics.OpenReplicationConnectionsGaugeName),
metric.WithDescription("Current open replication connections for PeerDB user"))
if err != nil {
logger.Error("Failed to get open replication connections gauge", slog.Any("error", err))
@@ -803,9 +801,8 @@ func (a *FlowableActivity) RecordSlotSizes(ctx context.Context) error {
}
slotMetricGauges.OpenReplicationConnectionsGauge = openReplicationConnectionsGauge
- intervalSinceLastNormalizeGauge, err := otel_metrics.GetOrInitFloat64SyncGauge(a.OtelManager.Meter,
- a.OtelManager.Float64GaugesCache,
- peerdb_gauges.BuildGaugeName(peerdb_gauges.IntervalSinceLastNormalizeGaugeName),
+ intervalSinceLastNormalizeGauge, err := a.OtelManager.GetOrInitFloat64Gauge(
+ otel_metrics.BuildMetricName(otel_metrics.IntervalSinceLastNormalizeGaugeName),
metric.WithUnit("s"),
metric.WithDescription("Interval since last normalize"))
if err != nil {
diff --git a/flow/activities/flowable_core.go b/flow/activities/flowable_core.go
index 64e6494caf..7d0d8cf022 100644
--- a/flow/activities/flowable_core.go
+++ b/flow/activities/flowable_core.go
@@ -23,6 +23,7 @@ import (
"github.com/PeerDB-io/peer-flow/connectors/utils/monitoring"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
+ "github.com/PeerDB-io/peer-flow/otel_metrics"
"github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
)
@@ -113,7 +114,7 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon
options *protos.SyncFlowOptions,
sessionID string,
adaptStream func(*model.CDCStream[Items]) (*model.CDCStream[Items], error),
- pull func(TPull, context.Context, *pgxpool.Pool, *model.PullRecordsRequest[Items]) error,
+ pull func(TPull, context.Context, *pgxpool.Pool, *otel_metrics.OtelManager, *model.PullRecordsRequest[Items]) error,
sync func(TSync, context.Context, *model.SyncRecordsRequest[Items]) (*model.SyncResponse, error),
) (*model.SyncCompositeResponse, error) {
flowName := config.FlowJobName
@@ -139,7 +140,7 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon
batchSize := options.BatchSize
if batchSize == 0 {
- batchSize = 1_000_000
+ batchSize = 250_000
}
lastOffset, err := func() (int64, error) {
@@ -181,7 +182,7 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon
startTime := time.Now()
errGroup, errCtx := errgroup.WithContext(ctx)
errGroup.Go(func() error {
- return pull(srcConn, errCtx, a.CatalogPool, &model.PullRecordsRequest[Items]{
+ return pull(srcConn, errCtx, a.CatalogPool, a.OtelManager, &model.PullRecordsRequest[Items]{
FlowJobName: flowName,
SrcTableIDNameMapping: options.SrcTableIdNameMapping,
TableNameMapping: tblNameMapping,
@@ -224,7 +225,7 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon
}
defer connectors.CloseConnector(ctx, dstConn)
- if err := dstConn.ReplayTableSchemaDeltas(ctx, flowName, recordBatchSync.SchemaDeltas); err != nil {
+ if err := dstConn.ReplayTableSchemaDeltas(ctx, config.Env, flowName, recordBatchSync.SchemaDeltas); err != nil {
return nil, fmt.Errorf("failed to sync schema: %w", err)
}
@@ -444,6 +445,7 @@ func replicateQRepPartition[TRead any, TWrite any, TSync connectors.QRepSyncConn
})
errGroup.Go(func() error {
+ var err error
rowsSynced, err = syncRecords(dstConn, errCtx, config, partition, outstream)
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
diff --git a/flow/activities/maintenance_activity.go b/flow/activities/maintenance_activity.go
new file mode 100644
index 0000000000..be42cc8e56
--- /dev/null
+++ b/flow/activities/maintenance_activity.go
@@ -0,0 +1,284 @@
+package activities
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+ "go.temporal.io/sdk/activity"
+ "go.temporal.io/sdk/client"
+ "google.golang.org/protobuf/types/known/timestamppb"
+
+ "github.com/PeerDB-io/peer-flow/alerting"
+ "github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/model"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
+ "github.com/PeerDB-io/peer-flow/shared"
+ "github.com/PeerDB-io/peer-flow/shared/telemetry"
+)
+
+const (
+ mirrorStateBackup = "backup"
+ mirrorStateRestored = "restore"
+)
+
+type MaintenanceActivity struct {
+ CatalogPool *pgxpool.Pool
+ Alerter *alerting.Alerter
+ TemporalClient client.Client
+}
+
+func (a *MaintenanceActivity) GetAllMirrors(ctx context.Context) (*protos.MaintenanceMirrors, error) {
+ rows, err := a.CatalogPool.Query(ctx, `
+ select distinct on(name)
+ id, name, workflow_id,
+ created_at, coalesce(query_string, '')='' is_cdc
+ from flows
+ `)
+ if err != nil {
+ return &protos.MaintenanceMirrors{}, err
+ }
+
+ maintenanceMirrorItems, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*protos.MaintenanceMirror, error) {
+ var info protos.MaintenanceMirror
+ var createdAt time.Time
+ err := row.Scan(&info.MirrorId, &info.MirrorName, &info.WorkflowId, &createdAt, &info.IsCdc)
+ info.MirrorCreatedAt = timestamppb.New(createdAt)
+ return &info, err
+ })
+ return &protos.MaintenanceMirrors{
+ Mirrors: maintenanceMirrorItems,
+ }, err
+}
+
+func (a *MaintenanceActivity) getMirrorStatus(ctx context.Context, mirror *protos.MaintenanceMirror) (protos.FlowStatus, error) {
+ return shared.GetWorkflowStatus(ctx, a.TemporalClient, mirror.WorkflowId)
+}
+
+func (a *MaintenanceActivity) WaitForRunningSnapshots(ctx context.Context) (*protos.MaintenanceMirrors, error) {
+ mirrors, err := a.GetAllMirrors(ctx)
+ if err != nil {
+ return &protos.MaintenanceMirrors{}, err
+ }
+
+ slog.Info("Found mirrors for snapshot check", "mirrors", mirrors, "len", len(mirrors.Mirrors))
+
+ for _, mirror := range mirrors.Mirrors {
+ lastStatus, err := a.checkAndWaitIfSnapshot(ctx, mirror, 2*time.Minute)
+ if err != nil {
+ return &protos.MaintenanceMirrors{}, err
+ }
+ slog.Info("Finished checking and waiting for snapshot",
+ "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId, "lastStatus", lastStatus.String())
+ }
+ slog.Info("Finished checking and waiting for all mirrors to finish snapshot")
+ return mirrors, nil
+}
+
+func (a *MaintenanceActivity) checkAndWaitIfSnapshot(
+ ctx context.Context,
+ mirror *protos.MaintenanceMirror,
+ logEvery time.Duration,
+) (protos.FlowStatus, error) {
+ // In case a mirror was just kicked off, it shows up in the running state, we wait for a bit before checking for snapshot
+ if mirror.MirrorCreatedAt.AsTime().After(time.Now().Add(-30 * time.Second)) {
+ slog.Info("Mirror was created less than 30 seconds ago, waiting for it to be ready before checking for snapshot",
+ "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId)
+ time.Sleep(30 * time.Second)
+ }
+
+ flowStatus, err := RunEveryIntervalUntilFinish(ctx, func() (bool, protos.FlowStatus, error) {
+ activity.RecordHeartbeat(ctx, fmt.Sprintf("Waiting for mirror %s to finish snapshot", mirror.MirrorName))
+ mirrorStatus, err := a.getMirrorStatus(ctx, mirror)
+ if err != nil {
+ return false, mirrorStatus, err
+ }
+ if mirrorStatus == protos.FlowStatus_STATUS_SNAPSHOT || mirrorStatus == protos.FlowStatus_STATUS_SETUP {
+ return false, mirrorStatus, nil
+ }
+ return true, mirrorStatus, nil
+ }, 10*time.Second, fmt.Sprintf("Waiting for mirror %s to finish snapshot", mirror.MirrorName), logEvery)
+ return flowStatus, err
+}
+
+func (a *MaintenanceActivity) EnableMaintenanceMode(ctx context.Context) error {
+ slog.Info("Enabling maintenance mode")
+ return peerdbenv.UpdatePeerDBMaintenanceModeEnabled(ctx, a.CatalogPool, true)
+}
+
+func (a *MaintenanceActivity) BackupAllPreviouslyRunningFlows(ctx context.Context, mirrors *protos.MaintenanceMirrors) error {
+ tx, err := a.CatalogPool.Begin(ctx)
+ if err != nil {
+ return err
+ }
+ defer shared.RollbackTx(tx, slog.Default())
+
+ for _, mirror := range mirrors.Mirrors {
+ _, err := tx.Exec(ctx, `
+ insert into maintenance.maintenance_flows
+ (flow_id, flow_name, workflow_id, flow_created_at, is_cdc, state, from_version)
+ values
+ ($1, $2, $3, $4, $5, $6, $7)
+ `, mirror.MirrorId, mirror.MirrorName, mirror.WorkflowId, mirror.MirrorCreatedAt.AsTime(), mirror.IsCdc, mirrorStateBackup,
+ peerdbenv.PeerDBVersionShaShort())
+ if err != nil {
+ return err
+ }
+ }
+ return tx.Commit(ctx)
+}
+
+func (a *MaintenanceActivity) PauseMirrorIfRunning(ctx context.Context, mirror *protos.MaintenanceMirror) (bool, error) {
+ mirrorStatus, err := a.getMirrorStatus(ctx, mirror)
+ if err != nil {
+ return false, err
+ }
+
+ slog.Info("Checking if mirror is running", "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId, "status", mirrorStatus.String())
+
+ if mirrorStatus != protos.FlowStatus_STATUS_RUNNING {
+ return false, nil
+ }
+
+ slog.Info("Pausing mirror for maintenance", "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId)
+
+ if err := model.FlowSignal.SignalClientWorkflow(ctx, a.TemporalClient, mirror.WorkflowId, "", model.PauseSignal); err != nil {
+ slog.Error("Error signaling mirror running to pause for maintenance",
+ "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId, "error", err)
+ return false, err
+ }
+
+ return RunEveryIntervalUntilFinish(ctx, func() (bool, bool, error) {
+ updatedMirrorStatus, statusErr := a.getMirrorStatus(ctx, mirror)
+ if statusErr != nil {
+ return false, false, statusErr
+ }
+ activity.RecordHeartbeat(ctx, "Waiting for mirror to pause with current status "+updatedMirrorStatus.String())
+ if statusErr := model.FlowSignal.SignalClientWorkflow(ctx, a.TemporalClient, mirror.WorkflowId, "",
+ model.PauseSignal); statusErr != nil {
+ return false, false, statusErr
+ }
+ if updatedMirrorStatus == protos.FlowStatus_STATUS_PAUSED {
+ return true, true, nil
+ }
+ return false, false, nil
+ }, 10*time.Second, "Waiting for mirror to pause", 30*time.Second)
+}
+
+func (a *MaintenanceActivity) CleanBackedUpFlows(ctx context.Context) error {
+ _, err := a.CatalogPool.Exec(ctx, `
+ update maintenance.maintenance_flows
+ set state = $1,
+ restored_at = now(),
+ to_version = $2
+ where state = $3
+ `, mirrorStateRestored, peerdbenv.PeerDBVersionShaShort(), mirrorStateBackup)
+ return err
+}
+
+func (a *MaintenanceActivity) GetBackedUpFlows(ctx context.Context) (*protos.MaintenanceMirrors, error) {
+ rows, err := a.CatalogPool.Query(ctx, `
+ select flow_id, flow_name, workflow_id, flow_created_at, is_cdc
+ from maintenance.maintenance_flows
+ where state = $1
+ `, mirrorStateBackup)
+ if err != nil {
+ return nil, err
+ }
+
+ maintenanceMirrorItems, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*protos.MaintenanceMirror, error) {
+ var info protos.MaintenanceMirror
+ var createdAt time.Time
+ err := row.Scan(&info.MirrorId, &info.MirrorName, &info.WorkflowId, &createdAt, &info.IsCdc)
+ info.MirrorCreatedAt = timestamppb.New(createdAt)
+ return &info, err
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ return &protos.MaintenanceMirrors{
+ Mirrors: maintenanceMirrorItems,
+ }, nil
+}
+
+func (a *MaintenanceActivity) ResumeMirror(ctx context.Context, mirror *protos.MaintenanceMirror) error {
+ mirrorStatus, err := a.getMirrorStatus(ctx, mirror)
+ if err != nil {
+ return err
+ }
+
+ if mirrorStatus != protos.FlowStatus_STATUS_PAUSED {
+ slog.Error("Cannot resume mirror that is not paused",
+ "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId, "status", mirrorStatus.String())
+ return nil
+ }
+
+ // There can also be "workflow already completed" errors, what should we do in that case?
+ if err := model.FlowSignal.SignalClientWorkflow(ctx, a.TemporalClient, mirror.WorkflowId, "", model.NoopSignal); err != nil {
+ slog.Error("Error signaling mirror to resume for maintenance",
+ "mirror", mirror.MirrorName, "workflowId", mirror.WorkflowId, "error", err)
+ return err
+ }
+ return nil
+}
+
+func (a *MaintenanceActivity) DisableMaintenanceMode(ctx context.Context) error {
+ slog.Info("Disabling maintenance mode")
+ return peerdbenv.UpdatePeerDBMaintenanceModeEnabled(ctx, a.CatalogPool, false)
+}
+
+func (a *MaintenanceActivity) BackgroundAlerter(ctx context.Context) error {
+ heartbeatTicker := time.NewTicker(30 * time.Second)
+ defer heartbeatTicker.Stop()
+
+ alertTicker := time.NewTicker(time.Duration(peerdbenv.PeerDBMaintenanceModeWaitAlertSeconds()) * time.Second)
+ defer alertTicker.Stop()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-heartbeatTicker.C:
+ activity.RecordHeartbeat(ctx, "Maintenance Workflow is still running")
+ case <-alertTicker.C:
+ slog.Warn("Maintenance Workflow is still running")
+ a.Alerter.LogNonFlowWarning(ctx, telemetry.MaintenanceWait, "Waiting", "Maintenance mode is still running")
+ }
+ }
+}
+
+func RunEveryIntervalUntilFinish[T any](
+ ctx context.Context,
+ runFunc func() (finished bool, result T, err error),
+ runInterval time.Duration,
+ logMessage string,
+ logInterval time.Duration,
+) (T, error) {
+ runTicker := time.NewTicker(runInterval)
+ defer runTicker.Stop()
+
+ logTicker := time.NewTicker(logInterval)
+ defer logTicker.Stop()
+ var lastResult T
+ for {
+ select {
+ case <-ctx.Done():
+ return lastResult, ctx.Err()
+ case <-runTicker.C:
+ finished, result, err := runFunc()
+ lastResult = result
+ if err != nil {
+ return lastResult, err
+ }
+ if finished {
+ return lastResult, err
+ }
+ case <-logTicker.C:
+ slog.Info(logMessage, "lastResult", lastResult)
+ }
+ }
+}
diff --git a/flow/alerting/alerting.go b/flow/alerting/alerting.go
index e9df410f91..d1394561fd 100644
--- a/flow/alerting/alerting.go
+++ b/flow/alerting/alerting.go
@@ -21,6 +21,7 @@ import (
"github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
"github.com/PeerDB-io/peer-flow/shared/telemetry"
+ "github.com/PeerDB-io/peer-flow/tags"
)
// alerting service, no cool name :(
@@ -356,7 +357,7 @@ func (a *Alerter) checkAndAddAlertToCatalog(ctx context.Context, alertConfigId i
return true
}
- logger.Info(fmt.Sprintf("Skipped sending alerts: last alert was sent at %s, which was >=%s ago", createdTimestamp.String(), dur.String()))
+ logger.Info(fmt.Sprintf("Skipped sending alerts: last alert was sent at %s, which was <=%s ago", createdTimestamp.String(), dur.String()))
return false
}
@@ -366,21 +367,32 @@ func (a *Alerter) sendTelemetryMessage(
flowName string,
more string,
level telemetry.Level,
- tags ...string,
+ additionalTags ...string,
) {
+ allTags := []string{flowName, peerdbenv.PeerDBDeploymentUID()}
+ allTags = append(allTags, additionalTags...)
+
+ if flowTags, err := tags.GetTags(ctx, a.CatalogPool, flowName); err != nil {
+ logger.Warn("failed to get flow tags", slog.Any("error", err))
+ } else {
+ for key, value := range flowTags {
+ allTags = append(allTags, fmt.Sprintf("%s:%s", key, value))
+ }
+ }
+
details := fmt.Sprintf("[%s] %s", flowName, more)
attributes := telemetry.Attributes{
Level: level,
DeploymentUID: peerdbenv.PeerDBDeploymentUID(),
- Tags: append([]string{flowName, peerdbenv.PeerDBDeploymentUID()}, tags...),
+ Tags: allTags,
Type: flowName,
}
if a.snsTelemetrySender != nil {
- if status, err := a.snsTelemetrySender.SendMessage(ctx, details, details, attributes); err != nil {
+ if response, err := a.snsTelemetrySender.SendMessage(ctx, details, details, attributes); err != nil {
logger.Warn("failed to send message to snsTelemetrySender", slog.Any("error", err))
} else {
- logger.Info("received status from snsTelemetrySender", slog.String("status", status))
+ logger.Info("received response from snsTelemetrySender", slog.String("response", response))
}
}
@@ -388,7 +400,7 @@ func (a *Alerter) sendTelemetryMessage(
if status, err := a.incidentIoTelemetrySender.SendMessage(ctx, details, details, attributes); err != nil {
logger.Warn("failed to send message to incidentIoTelemetrySender", slog.Any("error", err))
} else {
- logger.Info("received status from incident.io", slog.String("status", status))
+ logger.Info("received response from incident.io", slog.String("response", status))
}
}
}
@@ -440,6 +452,10 @@ func (a *Alerter) LogFlowError(ctx context.Context, flowName string, err error)
if errors.As(err, &pgErr) {
tags = append(tags, "pgcode:"+pgErr.Code)
}
+ var netErr *net.OpError
+ if errors.As(err, &netErr) {
+ tags = append(tags, "err:Net")
+ }
a.sendTelemetryMessage(ctx, logger, flowName, errorWithStack, telemetry.ERROR, tags...)
}
diff --git a/flow/cmd/api.go b/flow/cmd/api.go
index ca225e4292..f81f9d923d 100644
--- a/flow/cmd/api.go
+++ b/flow/cmd/api.go
@@ -191,24 +191,7 @@ func APIMain(ctx context.Context, args *APIServerParams) error {
Logger: slog.New(shared.NewSlogHandler(slog.NewJSONHandler(os.Stdout, nil))),
}
- if peerdbenv.PeerDBTemporalEnableCertAuth() {
- slog.Info("Using temporal certificate/key for authentication")
-
- certs, err := parseTemporalCertAndKey(ctx)
- if err != nil {
- return fmt.Errorf("unable to base64 decode certificate and key: %w", err)
- }
-
- connOptions := client.ConnectionOptions{
- TLS: &tls.Config{
- Certificates: certs,
- MinVersion: tls.VersionTLS13,
- },
- }
- clientOptions.ConnectionOptions = connOptions
- }
-
- tc, err := client.Dial(clientOptions)
+ tc, err := setupTemporalClient(ctx, clientOptions)
if err != nil {
return fmt.Errorf("unable to create Temporal client: %w", err)
}
@@ -309,3 +292,25 @@ func APIMain(ctx context.Context, args *APIServerParams) error {
return nil
}
+
+func setupTemporalClient(ctx context.Context, clientOptions client.Options) (client.Client, error) {
+ if peerdbenv.PeerDBTemporalEnableCertAuth() {
+ slog.Info("Using temporal certificate/key for authentication")
+
+ certs, err := parseTemporalCertAndKey(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("unable to base64 decode certificate and key: %w", err)
+ }
+
+ connOptions := client.ConnectionOptions{
+ TLS: &tls.Config{
+ Certificates: certs,
+ MinVersion: tls.VersionTLS13,
+ },
+ }
+ clientOptions.ConnectionOptions = connOptions
+ }
+
+ tc, err := client.Dial(clientOptions)
+ return tc, err
+}
diff --git a/flow/cmd/handler.go b/flow/cmd/handler.go
index e2d1da2e39..6caefaf47e 100644
--- a/flow/cmd/handler.go
+++ b/flow/cmd/handler.go
@@ -19,6 +19,7 @@ import (
"github.com/PeerDB-io/peer-flow/connectors/utils"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
peerflow "github.com/PeerDB-io/peer-flow/workflows"
)
@@ -327,6 +328,17 @@ func (h *FlowRequestHandler) FlowStateChange(
) (*protos.FlowStateChangeResponse, error) {
logs := slog.String("flowJobName", req.FlowJobName)
slog.Info("FlowStateChange called", logs, slog.Any("req", req))
+ underMaintenance, err := peerdbenv.PeerDBMaintenanceModeEnabled(ctx, nil)
+ if err != nil {
+ slog.Error("unable to check maintenance mode", logs, slog.Any("error", err))
+ return nil, fmt.Errorf("unable to load dynamic config: %w", err)
+ }
+
+ if underMaintenance {
+ slog.Warn("Flow state change request denied due to maintenance", logs)
+ return nil, errors.New("PeerDB is under maintenance")
+ }
+
workflowID, err := h.getWorkflowID(ctx, req.FlowJobName)
if err != nil {
slog.Error("[flow-state-change] unable to get workflowID", logs, slog.Any("error", err))
@@ -488,6 +500,14 @@ func (h *FlowRequestHandler) ResyncMirror(
ctx context.Context,
req *protos.ResyncMirrorRequest,
) (*protos.ResyncMirrorResponse, error) {
+ underMaintenance, err := peerdbenv.PeerDBMaintenanceModeEnabled(ctx, nil)
+ if err != nil {
+ return nil, fmt.Errorf("unable to get maintenance mode status: %w", err)
+ }
+ if underMaintenance {
+ return nil, errors.New("PeerDB is under maintenance")
+ }
+
isCDC, err := h.isCDCFlow(ctx, req.FlowJobName)
if err != nil {
return nil, err
@@ -521,3 +541,49 @@ func (h *FlowRequestHandler) ResyncMirror(
}
return &protos.ResyncMirrorResponse{}, nil
}
+
+func (h *FlowRequestHandler) GetInstanceInfo(ctx context.Context, in *protos.InstanceInfoRequest) (*protos.InstanceInfoResponse, error) {
+ enabled, err := peerdbenv.PeerDBMaintenanceModeEnabled(ctx, nil)
+ if err != nil {
+ slog.Error("unable to get maintenance mode status", slog.Any("error", err))
+ return &protos.InstanceInfoResponse{
+ Status: protos.InstanceStatus_INSTANCE_STATUS_UNKNOWN,
+ }, fmt.Errorf("unable to get maintenance mode status: %w", err)
+ }
+ if enabled {
+ return &protos.InstanceInfoResponse{
+ Status: protos.InstanceStatus_INSTANCE_STATUS_MAINTENANCE,
+ }, nil
+ }
+ return &protos.InstanceInfoResponse{
+ Status: protos.InstanceStatus_INSTANCE_STATUS_READY,
+ }, nil
+}
+
+func (h *FlowRequestHandler) Maintenance(ctx context.Context, in *protos.MaintenanceRequest) (*protos.MaintenanceResponse, error) {
+ taskQueueId := shared.MaintenanceFlowTaskQueue
+ if in.UsePeerflowTaskQueue {
+ taskQueueId = shared.PeerFlowTaskQueue
+ }
+ switch {
+ case in.Status == protos.MaintenanceStatus_MAINTENANCE_STATUS_START:
+ workflowRun, err := peerflow.RunStartMaintenanceWorkflow(ctx, h.temporalClient, &protos.StartMaintenanceFlowInput{}, taskQueueId)
+ if err != nil {
+ return nil, err
+ }
+ return &protos.MaintenanceResponse{
+ WorkflowId: workflowRun.GetID(),
+ RunId: workflowRun.GetRunID(),
+ }, nil
+ case in.Status == protos.MaintenanceStatus_MAINTENANCE_STATUS_END:
+ workflowRun, err := peerflow.RunEndMaintenanceWorkflow(ctx, h.temporalClient, &protos.EndMaintenanceFlowInput{}, taskQueueId)
+ if err != nil {
+ return nil, err
+ }
+ return &protos.MaintenanceResponse{
+ WorkflowId: workflowRun.GetID(),
+ RunId: workflowRun.GetRunID(),
+ }, nil
+ }
+ return nil, errors.New("invalid maintenance status")
+}
diff --git a/flow/cmd/maintenance.go b/flow/cmd/maintenance.go
new file mode 100644
index 0000000000..474a67db37
--- /dev/null
+++ b/flow/cmd/maintenance.go
@@ -0,0 +1,246 @@
+package cmd
+
+import (
+ "context"
+ "crypto/tls"
+ "errors"
+ "fmt"
+ "log/slog"
+ "os"
+
+ "github.com/aws/smithy-go/ptr"
+ "go.temporal.io/sdk/client"
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/credentials/insecure"
+ k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+ v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
+
+ "github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
+ "github.com/PeerDB-io/peer-flow/shared"
+ peerflow "github.com/PeerDB-io/peer-flow/workflows"
+)
+
+type MaintenanceCLIParams struct {
+ TemporalHostPort string
+ TemporalNamespace string
+ Mode string
+ FlowGrpcAddress string
+ SkipIfK8sServiceMissing string
+ FlowTlsEnabled bool
+ SkipOnApiVersionMatch bool
+ SkipOnNoMirrors bool
+ UseMaintenanceTaskQueue bool
+ AssumeSkippedMaintenanceWorkflows bool
+}
+
+type StartMaintenanceResult struct {
+ SkippedReason *string `json:"skippedReason,omitempty"`
+ APIVersion string `json:"apiVersion,omitempty"`
+ CLIVersion string `json:"cliVersion,omitempty"`
+ Skipped bool `json:"skipped,omitempty"`
+}
+
+// MaintenanceMain is the entry point for the maintenance command, requires access to Temporal client, will exit after
+// running the requested maintenance workflow
+func MaintenanceMain(ctx context.Context, args *MaintenanceCLIParams) error {
+ slog.Info("Starting Maintenance Mode CLI")
+ clientOptions := client.Options{
+ HostPort: args.TemporalHostPort,
+ Namespace: args.TemporalNamespace,
+ Logger: slog.New(shared.NewSlogHandler(slog.NewJSONHandler(os.Stdout, nil))),
+ }
+ tc, err := setupTemporalClient(ctx, clientOptions)
+ if err != nil {
+ return fmt.Errorf("unable to create Temporal client: %w", err)
+ }
+
+ taskQueueId := shared.MaintenanceFlowTaskQueue
+ if !args.UseMaintenanceTaskQueue {
+ taskQueueId = shared.PeerFlowTaskQueue
+ }
+
+ if args.Mode == "start" {
+ if args.AssumeSkippedMaintenanceWorkflows {
+ slog.Info("Assuming maintenance workflows were skipped")
+ return WriteMaintenanceOutputToCatalog(ctx, StartMaintenanceResult{
+ Skipped: true,
+ SkippedReason: ptr.String("Assumed skipped by CLI Flag"),
+ CLIVersion: peerdbenv.PeerDBVersionShaShort(),
+ })
+ }
+ skipped, err := skipStartMaintenanceIfNeeded(ctx, args)
+ if err != nil {
+ return err
+ }
+ if skipped {
+ return nil
+ }
+ slog.Info("Running start maintenance workflow")
+ workflowRun, err := peerflow.RunStartMaintenanceWorkflow(ctx, tc, &protos.StartMaintenanceFlowInput{}, taskQueueId)
+ if err != nil {
+ slog.Error("Error running start maintenance workflow", "error", err)
+ return err
+ }
+ var output *protos.StartMaintenanceFlowOutput
+ if err := workflowRun.Get(ctx, &output); err != nil {
+ slog.Error("Error in start maintenance workflow", "error", err)
+ return err
+ }
+ slog.Info("Start maintenance workflow completed", "output", output)
+ return WriteMaintenanceOutputToCatalog(ctx, StartMaintenanceResult{
+ Skipped: false,
+ CLIVersion: peerdbenv.PeerDBVersionShaShort(),
+ })
+ } else if args.Mode == "end" {
+ if input, err := ReadLastMaintenanceOutput(ctx); input != nil || err != nil {
+ if err != nil {
+ return err
+ }
+ slog.Info("Checking if end maintenance workflow should be skipped", "input", input)
+ if input.Skipped {
+ slog.Info("Skipping end maintenance workflow as start maintenance was skipped", "reason", input.SkippedReason)
+ return nil
+ }
+ }
+ workflowRun, err := peerflow.RunEndMaintenanceWorkflow(ctx, tc, &protos.EndMaintenanceFlowInput{}, taskQueueId)
+ if err != nil {
+ slog.Error("Error running end maintenance workflow", "error", err)
+ return err
+ }
+ var output *protos.EndMaintenanceFlowOutput
+ if err := workflowRun.Get(ctx, &output); err != nil {
+ slog.Error("Error in end maintenance workflow", "error", err)
+ return err
+ }
+ slog.Info("End maintenance workflow completed", "output", output)
+ } else {
+ return fmt.Errorf("unknown flow type %s", args.Mode)
+ }
+ slog.Info("Maintenance workflow completed with type", "type", args.Mode)
+ return nil
+}
+
+func skipStartMaintenanceIfNeeded(ctx context.Context, args *MaintenanceCLIParams) (bool, error) {
+ if args.SkipIfK8sServiceMissing != "" {
+ slog.Info("Checking if k8s service exists", "service", args.SkipIfK8sServiceMissing)
+ exists, err := CheckK8sServiceExistence(ctx, args.SkipIfK8sServiceMissing)
+ if err != nil {
+ return false, err
+ }
+ if !exists {
+ slog.Info("Skipping maintenance workflow due to missing k8s service", "service", args.SkipIfK8sServiceMissing)
+ return true, WriteMaintenanceOutputToCatalog(ctx, StartMaintenanceResult{
+ Skipped: true,
+ SkippedReason: ptr.String(fmt.Sprintf("K8s service %s missing", args.SkipIfK8sServiceMissing)),
+ CLIVersion: peerdbenv.PeerDBVersionShaShort(),
+ })
+ }
+ }
+ if args.SkipOnApiVersionMatch || args.SkipOnNoMirrors {
+ if args.FlowGrpcAddress == "" {
+ return false, errors.New("flow address is required when skipping based on API")
+ }
+ slog.Info("Constructing flow client")
+ transportCredentials := credentials.NewTLS(&tls.Config{
+ MinVersion: tls.VersionTLS12,
+ })
+ if !args.FlowTlsEnabled {
+ transportCredentials = insecure.NewCredentials()
+ }
+ conn, err := grpc.NewClient(args.FlowGrpcAddress,
+ grpc.WithTransportCredentials(transportCredentials),
+ )
+ if err != nil {
+ return false, fmt.Errorf("unable to dial grpc flow server: %w", err)
+ }
+ peerFlowClient := protos.NewFlowServiceClient(conn)
+ if args.SkipOnApiVersionMatch {
+ slog.Info("Checking if CLI version matches API version", "cliVersion", peerdbenv.PeerDBVersionShaShort())
+ version, err := peerFlowClient.GetVersion(ctx, &protos.PeerDBVersionRequest{})
+ if err != nil {
+ return false, err
+ }
+ slog.Info("Got version from flow", "version", version.Version)
+ if version.Version == peerdbenv.PeerDBVersionShaShort() {
+ slog.Info("Skipping maintenance workflow due to matching versions")
+ return true, WriteMaintenanceOutputToCatalog(ctx, StartMaintenanceResult{
+ Skipped: true,
+ SkippedReason: ptr.String(fmt.Sprintf("CLI version %s matches API version %s", peerdbenv.PeerDBVersionShaShort(),
+ version.Version)),
+ APIVersion: version.Version,
+ CLIVersion: peerdbenv.PeerDBVersionShaShort(),
+ })
+ }
+ }
+ if args.SkipOnNoMirrors {
+ slog.Info("Checking if there are any mirrors")
+ mirrors, err := peerFlowClient.ListMirrors(ctx, &protos.ListMirrorsRequest{})
+ if err != nil {
+ return false, err
+ }
+ slog.Info("Got mirrors from flow", "mirrors", mirrors.Mirrors)
+ if len(mirrors.Mirrors) == 0 {
+ slog.Info("Skipping maintenance workflow due to no mirrors")
+ return true, WriteMaintenanceOutputToCatalog(ctx, StartMaintenanceResult{
+ Skipped: true,
+ SkippedReason: ptr.String("No mirrors found"),
+ })
+ }
+ }
+ }
+ return false, nil
+}
+
+func WriteMaintenanceOutputToCatalog(ctx context.Context, result StartMaintenanceResult) error {
+ pool, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx)
+ if err != nil {
+ return err
+ }
+ _, err = pool.Exec(ctx, `
+ insert into maintenance.start_maintenance_outputs
+ (cli_version, api_version, skipped, skipped_reason)
+ values
+ ($1, $2, $3, $4)
+ `, result.CLIVersion, result.APIVersion, result.Skipped, result.SkippedReason)
+ return err
+}
+
+func ReadLastMaintenanceOutput(ctx context.Context) (*StartMaintenanceResult, error) {
+ pool, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var result StartMaintenanceResult
+ if err := pool.QueryRow(ctx, `
+ select cli_version, api_version, skipped, skipped_reason
+ from maintenance.start_maintenance_outputs
+ order by created_at desc
+ limit 1
+ `).Scan(&result.CLIVersion, &result.APIVersion, &result.Skipped, &result.SkippedReason); err != nil {
+ return nil, err
+ }
+ return &result, nil
+}
+
+func CheckK8sServiceExistence(ctx context.Context, serviceName string) (bool, error) {
+ config, err := rest.InClusterConfig()
+ if err != nil {
+ return false, err
+ }
+ clientset, err := kubernetes.NewForConfig(config)
+ if err != nil {
+ return false, err
+ }
+ _, err = clientset.CoreV1().Services(peerdbenv.GetEnvString("POD_NAMESPACE", "")).Get(ctx, serviceName, v1.GetOptions{})
+ if err != nil {
+ if k8sErrors.IsNotFound(err) {
+ return false, nil
+ }
+ return false, err
+ }
+ return true, nil
+}
diff --git a/flow/cmd/mirror_status.go b/flow/cmd/mirror_status.go
index a0c4a989e2..156185054c 100644
--- a/flow/cmd/mirror_status.go
+++ b/flow/cmd/mirror_status.go
@@ -215,8 +215,8 @@ func (h *FlowRequestHandler) CDCGraph(ctx context.Context, req *protos.GraphRequ
}
rows, err := h.pool.Query(ctx, `select tm, coalesce(sum(rows_in_batch), 0)
from generate_series(date_trunc($2, now() - $1::INTERVAL * 30), now(), $1::INTERVAL) tm
- left join peerdb_stats.cdc_batches on start_time >= tm and start_time < tm + $1::INTERVAL
- group by 1 order by 1`, req.AggregateType, truncField)
+ left join peerdb_stats.cdc_batches on start_time >= tm and start_time < tm + $1::INTERVAL and flow_name = $3
+ group by 1 order by 1`, req.AggregateType, truncField, req.FlowJobName)
if err != nil {
return nil, err
}
@@ -447,20 +447,7 @@ func (h *FlowRequestHandler) isCDCFlow(ctx context.Context, flowJobName string)
}
func (h *FlowRequestHandler) getWorkflowStatus(ctx context.Context, workflowID string) (protos.FlowStatus, error) {
- res, err := h.temporalClient.QueryWorkflow(ctx, workflowID, "", shared.FlowStatusQuery)
- if err != nil {
- slog.Error(fmt.Sprintf("failed to get status in workflow with ID %s: %s", workflowID, err.Error()))
- return protos.FlowStatus_STATUS_UNKNOWN,
- fmt.Errorf("failed to get status in workflow with ID %s: %w", workflowID, err)
- }
- var state protos.FlowStatus
- err = res.Get(&state)
- if err != nil {
- slog.Error(fmt.Sprintf("failed to get status in workflow with ID %s: %s", workflowID, err.Error()))
- return protos.FlowStatus_STATUS_UNKNOWN,
- fmt.Errorf("failed to get status in workflow with ID %s: %w", workflowID, err)
- }
- return state, nil
+ return shared.GetWorkflowStatus(ctx, h.temporalClient, workflowID)
}
func (h *FlowRequestHandler) getCDCWorkflowState(ctx context.Context,
diff --git a/flow/cmd/settings.go b/flow/cmd/settings.go
index 12e0728590..dd4755f4ae 100644
--- a/flow/cmd/settings.go
+++ b/flow/cmd/settings.go
@@ -55,8 +55,7 @@ func (h *FlowRequestHandler) PostDynamicSetting(
ctx context.Context,
req *protos.PostDynamicSettingRequest,
) (*protos.PostDynamicSettingResponse, error) {
- _, err := h.pool.Exec(ctx, `insert into dynamic_settings (config_name, config_value) values ($1, $2)
- on conflict (config_name) do update set config_value = $2`, req.Name, req.Value)
+ err := peerdbenv.UpdateDynamicSetting(ctx, h.pool, req.Name, req.Value)
if err != nil {
slog.Error("[PostDynamicConfig] failed to execute update setting", slog.Any("error", err))
return nil, err
diff --git a/flow/cmd/tags_handler.go b/flow/cmd/tags_handler.go
new file mode 100644
index 0000000000..ddd362c3e4
--- /dev/null
+++ b/flow/cmd/tags_handler.go
@@ -0,0 +1,84 @@
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/tags"
+)
+
+func (h *FlowRequestHandler) flowExists(ctx context.Context, flowName string) (bool, error) {
+ var exists bool
+ err := h.pool.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM flows WHERE name = $1)", flowName).Scan(&exists)
+ if err != nil {
+ slog.Error("error checking if flow exists", slog.Any("error", err))
+ return false, err
+ }
+
+ slog.Info(fmt.Sprintf("flow %s exists: %t", flowName, exists))
+ return exists, nil
+}
+
+func (h *FlowRequestHandler) CreateOrReplaceFlowTags(
+ ctx context.Context,
+ in *protos.CreateOrReplaceFlowTagsRequest,
+) (*protos.CreateOrReplaceFlowTagsResponse, error) {
+ flowName := in.FlowName
+
+ exists, err := h.flowExists(ctx, flowName)
+ if err != nil {
+ return nil, err
+ }
+
+ if !exists {
+ slog.Error("flow does not exist", slog.String("flow_name", flowName))
+ return nil, fmt.Errorf("flow %s does not exist", flowName)
+ }
+
+ tags := make(map[string]string, len(in.Tags))
+ for _, tag := range in.Tags {
+ tags[tag.Key] = tag.Value
+ }
+
+ _, err = h.pool.Exec(ctx, "UPDATE flows SET tags = $1 WHERE name = $2", tags, flowName)
+ if err != nil {
+ slog.Error("error updating flow tags", slog.Any("error", err))
+ return nil, err
+ }
+
+ return &protos.CreateOrReplaceFlowTagsResponse{
+ FlowName: flowName,
+ }, nil
+}
+
+func (h *FlowRequestHandler) GetFlowTags(ctx context.Context, in *protos.GetFlowTagsRequest) (*protos.GetFlowTagsResponse, error) {
+ flowName := in.FlowName
+
+ exists, err := h.flowExists(ctx, flowName)
+ if err != nil {
+ return nil, err
+ }
+
+ if !exists {
+ slog.Error("flow does not exist", slog.String("flow_name", flowName))
+ return nil, fmt.Errorf("flow %s does not exist", flowName)
+ }
+
+ tags, err := tags.GetTags(ctx, h.pool, flowName)
+ if err != nil {
+ slog.Error("error getting flow tags", slog.Any("error", err))
+ return nil, err
+ }
+
+ protosTags := make([]*protos.FlowTag, 0, len(tags))
+ for key, value := range tags {
+ protosTags = append(protosTags, &protos.FlowTag{Key: key, Value: value})
+ }
+
+ return &protos.GetFlowTagsResponse{
+ FlowName: flowName,
+ Tags: protosTags,
+ }, nil
+}
diff --git a/flow/cmd/validate_mirror.go b/flow/cmd/validate_mirror.go
index 3e870aa667..83c9d2a073 100644
--- a/flow/cmd/validate_mirror.go
+++ b/flow/cmd/validate_mirror.go
@@ -14,6 +14,7 @@ import (
connpostgres "github.com/PeerDB-io/peer-flow/connectors/postgres"
"github.com/PeerDB-io/peer-flow/connectors/utils"
"github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared/telemetry"
)
@@ -25,6 +26,17 @@ var (
func (h *FlowRequestHandler) ValidateCDCMirror(
ctx context.Context, req *protos.CreateCDCFlowRequest,
) (*protos.ValidateCDCMirrorResponse, error) {
+ underMaintenance, err := peerdbenv.PeerDBMaintenanceModeEnabled(ctx, nil)
+ if err != nil {
+ slog.Error("unable to check maintenance mode", slog.Any("error", err))
+ return nil, fmt.Errorf("unable to load dynamic config: %w", err)
+ }
+
+ if underMaintenance {
+ slog.Warn("Validate request denied due to maintenance", "flowName", req.ConnectionConfigs.FlowJobName)
+ return nil, errors.New("PeerDB is under maintenance")
+ }
+
if !req.ConnectionConfigs.Resync {
mirrorExists, existCheckErr := h.CheckIfMirrorNameExists(ctx, req.ConnectionConfigs.FlowJobName)
if existCheckErr != nil {
diff --git a/flow/cmd/worker.go b/flow/cmd/worker.go
index 9db97288cc..87fbd0aa54 100644
--- a/flow/cmd/worker.go
+++ b/flow/cmd/worker.go
@@ -30,12 +30,22 @@ type WorkerSetupOptions struct {
TemporalMaxConcurrentWorkflowTasks int
EnableProfiling bool
EnableOtelMetrics bool
+ UseMaintenanceTaskQueue bool
}
type workerSetupResponse struct {
- Client client.Client
- Worker worker.Worker
- Cleanup func()
+ Client client.Client
+ Worker worker.Worker
+ OtelManager *otel_metrics.OtelManager
+}
+
+func (w *workerSetupResponse) Close() {
+ w.Client.Close()
+ if w.OtelManager != nil {
+ if err := w.OtelManager.Close(context.Background()); err != nil {
+ slog.Error("Failed to shutdown metrics provider", slog.Any("error", err))
+ }
+ }
}
func setupPyroscope(opts *WorkerSetupOptions) {
@@ -124,8 +134,11 @@ func WorkerSetup(opts *WorkerSetupOptions) (*workerSetupResponse, error) {
return nil, fmt.Errorf("unable to create Temporal client: %w", err)
}
slog.Info("Created temporal client")
-
- taskQueue := peerdbenv.PeerFlowTaskQueueName(shared.PeerFlowTaskQueue)
+ queueId := shared.PeerFlowTaskQueue
+ if opts.UseMaintenanceTaskQueue {
+ queueId = shared.MaintenanceFlowTaskQueue
+ }
+ taskQueue := peerdbenv.PeerFlowTaskQueueName(queueId)
slog.Info(
fmt.Sprintf("Creating temporal worker for queue %v: %v workflow workers %v activity workers",
taskQueue,
@@ -143,26 +156,14 @@ func WorkerSetup(opts *WorkerSetupOptions) (*workerSetupResponse, error) {
})
peerflow.RegisterFlowWorkerWorkflows(w)
- cleanupOtelManagerFunc := func() {}
var otelManager *otel_metrics.OtelManager
if opts.EnableOtelMetrics {
- metricsProvider, metricsErr := otel_metrics.SetupPeerDBMetricsProvider("flow-worker")
- if metricsErr != nil {
- return nil, metricsErr
- }
- otelManager = &otel_metrics.OtelManager{
- MetricsProvider: metricsProvider,
- Meter: metricsProvider.Meter("io.peerdb.flow-worker"),
- Float64GaugesCache: make(map[string]*otel_metrics.Float64SyncGauge),
- Int64GaugesCache: make(map[string]*otel_metrics.Int64SyncGauge),
- }
- cleanupOtelManagerFunc = func() {
- shutDownErr := otelManager.MetricsProvider.Shutdown(context.Background())
- if shutDownErr != nil {
- slog.Error("Failed to shutdown metrics provider", slog.Any("error", shutDownErr))
- }
+ otelManager, err = otel_metrics.NewOtelManager()
+ if err != nil {
+ return nil, fmt.Errorf("unable to create otel manager: %w", err)
}
}
+
w.RegisterActivity(&activities.FlowableActivity{
CatalogPool: conn,
Alerter: alerting.NewAlerter(context.Background(), conn),
@@ -170,12 +171,15 @@ func WorkerSetup(opts *WorkerSetupOptions) (*workerSetupResponse, error) {
OtelManager: otelManager,
})
+ w.RegisterActivity(&activities.MaintenanceActivity{
+ CatalogPool: conn,
+ Alerter: alerting.NewAlerter(context.Background(), conn),
+ TemporalClient: c,
+ })
+
return &workerSetupResponse{
- Client: c,
- Worker: w,
- Cleanup: func() {
- cleanupOtelManagerFunc()
- c.Close()
- },
+ Client: c,
+ Worker: w,
+ OtelManager: otelManager,
}, nil
}
diff --git a/flow/connectors/bigquery/bigquery.go b/flow/connectors/bigquery/bigquery.go
index f990b2f19d..d6504322ca 100644
--- a/flow/connectors/bigquery/bigquery.go
+++ b/flow/connectors/bigquery/bigquery.go
@@ -203,6 +203,7 @@ func (c *BigQueryConnector) waitForTableReady(ctx context.Context, datasetTable
// This could involve adding or dropping multiple columns.
func (c *BigQueryConnector) ReplayTableSchemaDeltas(
ctx context.Context,
+ env map[string]string,
flowJobName string,
schemaDeltas []*protos.TableSchemaDelta,
) error {
diff --git a/flow/connectors/bigquery/merge_stmt_generator.go b/flow/connectors/bigquery/merge_stmt_generator.go
index e903ef5869..5ee4f883c2 100644
--- a/flow/connectors/bigquery/merge_stmt_generator.go
+++ b/flow/connectors/bigquery/merge_stmt_generator.go
@@ -34,7 +34,7 @@ func (m *mergeStmtGenerator) generateFlattenedCTE(dstTable string, normalizedTab
var castStmt string
shortCol := m.shortColumn[column.Name]
switch qvalue.QValueKind(colType) {
- case qvalue.QValueKindJSON, qvalue.QValueKindHStore:
+ case qvalue.QValueKindJSON, qvalue.QValueKindJSONB, qvalue.QValueKindHStore:
// if the type is JSON, then just extract JSON
castStmt = fmt.Sprintf("CAST(PARSE_JSON(JSON_VALUE(_peerdb_data, '$.%s'),wide_number_mode=>'round') AS %s) AS `%s`",
column.Name, bqTypeString, shortCol)
diff --git a/flow/connectors/bigquery/qrep.go b/flow/connectors/bigquery/qrep.go
index 3da50c8e8f..b184cc62a9 100644
--- a/flow/connectors/bigquery/qrep.go
+++ b/flow/connectors/bigquery/qrep.go
@@ -35,7 +35,7 @@ func (c *BigQueryConnector) SyncQRepRecords(
partition.PartitionId, destTable))
avroSync := NewQRepAvroSyncMethod(c, config.StagingPath, config.FlowJobName)
- return avroSync.SyncQRepRecords(ctx, config.FlowJobName, destTable, partition,
+ return avroSync.SyncQRepRecords(ctx, config.Env, config.FlowJobName, destTable, partition,
tblMetadata, stream, config.SyncedAtColName, config.SoftDeleteColName)
}
@@ -80,7 +80,7 @@ func (c *BigQueryConnector) replayTableSchemaDeltasQRep(
}
}
- err = c.ReplayTableSchemaDeltas(ctx, config.FlowJobName, []*protos.TableSchemaDelta{tableSchemaDelta})
+ err = c.ReplayTableSchemaDeltas(ctx, config.Env, config.FlowJobName, []*protos.TableSchemaDelta{tableSchemaDelta})
if err != nil {
return nil, fmt.Errorf("failed to add columns to destination table: %w", err)
}
diff --git a/flow/connectors/bigquery/qrep_avro_sync.go b/flow/connectors/bigquery/qrep_avro_sync.go
index da3b15c37f..07285eb997 100644
--- a/flow/connectors/bigquery/qrep_avro_sync.go
+++ b/flow/connectors/bigquery/qrep_avro_sync.go
@@ -55,7 +55,7 @@ func (s *QRepAvroSyncMethod) SyncRecords(
}
stagingTable := fmt.Sprintf("%s_%s_staging", rawTableName, strconv.FormatInt(syncBatchID, 10))
- numRecords, err := s.writeToStage(ctx, strconv.FormatInt(syncBatchID, 10), rawTableName, avroSchema,
+ numRecords, err := s.writeToStage(ctx, req.Env, strconv.FormatInt(syncBatchID, 10), rawTableName, avroSchema,
&datasetTable{
project: s.connector.projectID,
dataset: s.connector.datasetID,
@@ -97,7 +97,7 @@ func (s *QRepAvroSyncMethod) SyncRecords(
slog.String(string(shared.FlowNameKey), req.FlowJobName),
slog.String("dstTableName", rawTableName))
- err = s.connector.ReplayTableSchemaDeltas(ctx, req.FlowJobName, req.Records.SchemaDeltas)
+ err = s.connector.ReplayTableSchemaDeltas(ctx, req.Env, req.FlowJobName, req.Records.SchemaDeltas)
if err != nil {
return nil, fmt.Errorf("failed to sync schema changes: %w", err)
}
@@ -139,6 +139,7 @@ func getTransformedColumns(dstSchema *bigquery.Schema, syncedAtCol string, softD
func (s *QRepAvroSyncMethod) SyncQRepRecords(
ctx context.Context,
+ env map[string]string,
flowJobName string,
dstTableName string,
partition *protos.QRepPartition,
@@ -167,7 +168,7 @@ func (s *QRepAvroSyncMethod) SyncQRepRecords(
table: fmt.Sprintf("%s_%s_staging", dstDatasetTable.table,
strings.ReplaceAll(partition.PartitionId, "-", "_")),
}
- numRecords, err := s.writeToStage(ctx, partition.PartitionId, flowJobName, avroSchema,
+ numRecords, err := s.writeToStage(ctx, env, partition.PartitionId, flowJobName, avroSchema,
stagingDatasetTable, stream, flowJobName)
if err != nil {
return -1, fmt.Errorf("failed to push to avro stage: %w", err)
@@ -389,6 +390,7 @@ func GetAvroField(bqField *bigquery.FieldSchema) (AvroField, error) {
func (s *QRepAvroSyncMethod) writeToStage(
ctx context.Context,
+ env map[string]string,
syncID string,
objectFolder string,
avroSchema *model.QRecordAvroSchemaDefinition,
@@ -408,7 +410,7 @@ func (s *QRepAvroSyncMethod) writeToStage(
obj := bucket.Object(avroFilePath)
w := obj.NewWriter(ctx)
- numRecords, err := ocfWriter.WriteOCF(ctx, w)
+ numRecords, err := ocfWriter.WriteOCF(ctx, env, w)
if err != nil {
return 0, fmt.Errorf("failed to write records to Avro file on GCS: %w", err)
}
@@ -426,7 +428,7 @@ func (s *QRepAvroSyncMethod) writeToStage(
avroFilePath := fmt.Sprintf("%s/%s.avro", tmpDir, syncID)
s.connector.logger.Info("writing records to local file", idLog)
- avroFile, err = ocfWriter.WriteRecordsToAvroFile(ctx, avroFilePath)
+ avroFile, err = ocfWriter.WriteRecordsToAvroFile(ctx, env, avroFilePath)
if err != nil {
return 0, fmt.Errorf("failed to write records to local Avro file: %w", err)
}
diff --git a/flow/connectors/bigquery/qvalue_convert.go b/flow/connectors/bigquery/qvalue_convert.go
index d2d9d9f0c2..aa798641ac 100644
--- a/flow/connectors/bigquery/qvalue_convert.go
+++ b/flow/connectors/bigquery/qvalue_convert.go
@@ -34,7 +34,7 @@ func qValueKindToBigQueryType(columnDescription *protos.FieldDescription, nullab
case qvalue.QValueKindString:
bqField.Type = bigquery.StringFieldType
// json also is stored as string for now
- case qvalue.QValueKindJSON, qvalue.QValueKindHStore:
+ case qvalue.QValueKindJSON, qvalue.QValueKindJSONB, qvalue.QValueKindHStore:
bqField.Type = bigquery.JSONFieldType
// time related
case qvalue.QValueKindTimestamp, qvalue.QValueKindTimestampTZ:
diff --git a/flow/connectors/clickhouse/cdc.go b/flow/connectors/clickhouse/cdc.go
index 3e002f5028..5dc8a14628 100644
--- a/flow/connectors/clickhouse/cdc.go
+++ b/flow/connectors/clickhouse/cdc.go
@@ -46,13 +46,13 @@ func (c *ClickHouseConnector) CreateRawTable(ctx context.Context, req *protos.Cr
rawTableName := c.getRawTableName(req.FlowJobName)
createRawTableSQL := `CREATE TABLE IF NOT EXISTS %s (
- _peerdb_uid UUID NOT NULL,
- _peerdb_timestamp Int64 NOT NULL,
- _peerdb_destination_table_name String NOT NULL,
- _peerdb_data String NOT NULL,
- _peerdb_record_type Int NOT NULL,
+ _peerdb_uid UUID,
+ _peerdb_timestamp Int64,
+ _peerdb_destination_table_name String,
+ _peerdb_data String,
+ _peerdb_record_type Int,
_peerdb_match_data String,
- _peerdb_batch_id Int,
+ _peerdb_batch_id Int64,
_peerdb_unchanged_toast_columns String
) ENGINE = MergeTree() ORDER BY (_peerdb_batch_id, _peerdb_destination_table_name);`
@@ -88,12 +88,12 @@ func (c *ClickHouseConnector) syncRecordsViaAvro(
}
avroSyncer := c.avroSyncMethod(req.FlowJobName)
- numRecords, err := avroSyncer.SyncRecords(ctx, stream, req.FlowJobName, syncBatchID)
+ numRecords, err := avroSyncer.SyncRecords(ctx, req.Env, stream, req.FlowJobName, syncBatchID)
if err != nil {
return nil, err
}
- if err := c.ReplayTableSchemaDeltas(ctx, req.FlowJobName, req.Records.SchemaDeltas); err != nil {
+ if err := c.ReplayTableSchemaDeltas(ctx, req.Env, req.FlowJobName, req.Records.SchemaDeltas); err != nil {
return nil, fmt.Errorf("failed to sync schema changes: %w", err)
}
@@ -120,7 +120,10 @@ func (c *ClickHouseConnector) SyncRecords(ctx context.Context, req *model.SyncRe
return res, nil
}
-func (c *ClickHouseConnector) ReplayTableSchemaDeltas(ctx context.Context, flowJobName string,
+func (c *ClickHouseConnector) ReplayTableSchemaDeltas(
+ ctx context.Context,
+ env map[string]string,
+ flowJobName string,
schemaDeltas []*protos.TableSchemaDelta,
) error {
if len(schemaDeltas) == 0 {
@@ -133,7 +136,7 @@ func (c *ClickHouseConnector) ReplayTableSchemaDeltas(ctx context.Context, flowJ
}
for _, addedColumn := range schemaDelta.AddedColumns {
- clickHouseColType, err := qvalue.QValueKind(addedColumn.Type).ToDWHColumnType(protos.DBType_CLICKHOUSE)
+ clickHouseColType, err := qvalue.QValueKind(addedColumn.Type).ToDWHColumnType(ctx, env, protos.DBType_CLICKHOUSE, addedColumn)
if err != nil {
return fmt.Errorf("failed to convert column type %s to ClickHouse type: %w", addedColumn.Type, err)
}
diff --git a/flow/connectors/clickhouse/clickhouse.go b/flow/connectors/clickhouse/clickhouse.go
index 4e89757014..f024a767e4 100644
--- a/flow/connectors/clickhouse/clickhouse.go
+++ b/flow/connectors/clickhouse/clickhouse.go
@@ -128,7 +128,7 @@ func NewClickHouseConnector(
config *protos.ClickhouseConfig,
) (*ClickHouseConnector, error) {
logger := shared.LoggerFromCtx(ctx)
- database, err := Connect(ctx, config)
+ database, err := Connect(ctx, env, config)
if err != nil {
return nil, fmt.Errorf("failed to open connection to ClickHouse peer: %w", err)
}
@@ -205,7 +205,7 @@ func NewClickHouseConnector(
return connector, nil
}
-func Connect(ctx context.Context, config *protos.ClickhouseConfig) (clickhouse.Conn, error) {
+func Connect(ctx context.Context, env map[string]string, config *protos.ClickhouseConfig) (clickhouse.Conn, error) {
var tlsSetting *tls.Config
if !config.DisableTls {
tlsSetting = &tls.Config{MinVersion: tls.VersionTLS13}
@@ -228,6 +228,14 @@ func Connect(ctx context.Context, config *protos.ClickhouseConfig) (clickhouse.C
tlsSetting.RootCAs = caPool
}
+ // See: https://clickhouse.com/docs/en/cloud/reference/shared-merge-tree#consistency
+ settings := clickhouse.Settings{"select_sequential_consistency": uint64(1)}
+ if maxInsertThreads, err := peerdbenv.PeerDBClickHouseMaxInsertThreads(ctx, env); err != nil {
+ return nil, fmt.Errorf("failed to load max_insert_threads config: %w", err)
+ } else if maxInsertThreads != 0 {
+ settings["max_insert_threads"] = maxInsertThreads
+ }
+
conn, err := clickhouse.Open(&clickhouse.Options{
Addr: []string{fmt.Sprintf("%s:%d", config.Host, config.Port)},
Auth: clickhouse.Auth{
@@ -245,6 +253,7 @@ func Connect(ctx context.Context, config *protos.ClickhouseConfig) (clickhouse.C
{Name: "peerdb"},
},
},
+ Settings: settings,
DialTimeout: 3600 * time.Second,
ReadTimeout: 3600 * time.Second,
})
diff --git a/flow/connectors/clickhouse/normalize.go b/flow/connectors/clickhouse/normalize.go
index 770abc7f20..fabe07a35f 100644
--- a/flow/connectors/clickhouse/normalize.go
+++ b/flow/connectors/clickhouse/normalize.go
@@ -12,7 +12,9 @@ import (
"strings"
"time"
- "github.com/PeerDB-io/peer-flow/datatypes"
+ "github.com/ClickHouse/clickhouse-go/v2"
+ "golang.org/x/sync/errgroup"
+
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/model/qvalue"
@@ -78,16 +80,6 @@ func getColName(overrides map[string]string, name string) string {
return name
}
-func getClickhouseTypeForNumericColumn(column *protos.FieldDescription) string {
- rawPrecision, _ := datatypes.ParseNumericTypmod(column.TypeModifier)
- if rawPrecision > datatypes.PeerDBClickHouseMaxPrecision {
- return "String"
- } else {
- precision, scale := datatypes.GetNumericTypeForWarehouse(column.TypeModifier, datatypes.ClickHouseNumericCompatibility{})
- return fmt.Sprintf("Decimal(%d, %d)", precision, scale)
- }
-}
-
func generateCreateTableSQLForNormalizedTable(
ctx context.Context,
config *protos.SetupNormalizedTableBatchInput,
@@ -139,14 +131,10 @@ func generateCreateTableSQLForNormalizedTable(
}
if clickHouseType == "" {
- if colType == qvalue.QValueKindNumeric {
- clickHouseType = getClickhouseTypeForNumericColumn(column)
- } else {
- var err error
- clickHouseType, err = colType.ToDWHColumnType(protos.DBType_CLICKHOUSE)
- if err != nil {
- return "", fmt.Errorf("error while converting column type to ClickHouse type: %w", err)
- }
+ var err error
+ clickHouseType, err = colType.ToDWHColumnType(ctx, config.Env, protos.DBType_CLICKHOUSE, column)
+ if err != nil {
+ return "", fmt.Errorf("error while converting column type to ClickHouse type: %w", err)
}
}
if (tableSchema.NullableEnabled || columnNullableEnabled) && column.Nullable && !colType.IsArray() {
@@ -262,8 +250,7 @@ func (c *ClickHouseConnector) NormalizeRecords(
}, nil
}
- err = c.copyAvroStagesToDestination(ctx, req.FlowJobName, normBatchID, req.SyncBatchID)
- if err != nil {
+ if err := c.copyAvroStagesToDestination(ctx, req.FlowJobName, normBatchID, req.SyncBatchID); err != nil {
return nil, fmt.Errorf("failed to copy avro stages to destination: %w", err)
}
@@ -278,9 +265,48 @@ func (c *ClickHouseConnector) NormalizeRecords(
return nil, err
}
+ enablePrimaryUpdate, err := peerdbenv.PeerDBEnableClickHousePrimaryUpdate(ctx, req.Env)
+ if err != nil {
+ return nil, err
+ }
+
+ parallelNormalize, err := peerdbenv.PeerDBClickHouseParallelNormalize(ctx, req.Env)
+ if err != nil {
+ return nil, err
+ }
+ parallelNormalize = min(max(parallelNormalize, 1), len(destinationTableNames))
+ if parallelNormalize > 1 {
+ c.logger.Info("normalizing in parallel", slog.Int("connections", parallelNormalize))
+ }
+
+ queries := make(chan string)
rawTbl := c.getRawTableName(req.FlowJobName)
- // model the raw table data as inserts.
+ group, errCtx := errgroup.WithContext(ctx)
+ for i := range parallelNormalize {
+ group.Go(func() error {
+ var chConn clickhouse.Conn
+ if i == 0 {
+ chConn = c.database
+ } else {
+ var err error
+ chConn, err = Connect(errCtx, req.Env, c.config)
+ if err != nil {
+ return err
+ }
+ defer chConn.Close()
+ }
+
+ for query := range queries {
+ c.logger.Info("normalizing batch", slog.String("query", query))
+ if err := chConn.Exec(errCtx, query); err != nil {
+ return fmt.Errorf("error while inserting into normalized table: %w", err)
+ }
+ }
+ return nil
+ })
+ }
+
for _, tbl := range destinationTableNames {
// SELECT projection FROM raw_table WHERE _peerdb_batch_id > normalize_batch_id AND _peerdb_batch_id <= sync_batch_id
selectQuery := strings.Builder{}
@@ -299,11 +325,6 @@ func (c *ClickHouseConnector) NormalizeRecords(
}
}
- enablePrimaryUpdate, err := peerdbenv.PeerDBEnableClickHousePrimaryUpdate(ctx, req.Env)
- if err != nil {
- return nil, err
- }
-
projection := strings.Builder{}
projectionUpdate := strings.Builder{}
@@ -332,15 +353,13 @@ func (c *ClickHouseConnector) NormalizeRecords(
colSelector.WriteString(fmt.Sprintf("`%s`,", dstColName))
if clickHouseType == "" {
- if colType == qvalue.QValueKindNumeric {
- clickHouseType = getClickhouseTypeForNumericColumn(column)
- } else {
- var err error
- clickHouseType, err = colType.ToDWHColumnType(protos.DBType_CLICKHOUSE)
- if err != nil {
- return nil, fmt.Errorf("error while converting column type to clickhouse type: %w", err)
- }
+ var err error
+ clickHouseType, err = colType.ToDWHColumnType(ctx, req.Env, protos.DBType_CLICKHOUSE, column)
+ if err != nil {
+ close(queries)
+ return nil, fmt.Errorf("error while converting column type to clickhouse type: %w", err)
}
+
if (schema.NullableEnabled || columnNullableEnabled) && column.Nullable && !colType.IsArray() {
clickHouseType = fmt.Sprintf("Nullable(%s)", clickHouseType)
}
@@ -433,15 +452,22 @@ func (c *ClickHouseConnector) NormalizeRecords(
insertIntoSelectQuery.WriteString(colSelector.String())
insertIntoSelectQuery.WriteString(selectQuery.String())
- q := insertIntoSelectQuery.String()
-
- if err := c.execWithLogging(ctx, q); err != nil {
- return nil, fmt.Errorf("error while inserting into normalized table: %w", err)
+ select {
+ case queries <- insertIntoSelectQuery.String():
+ case <-errCtx.Done():
+ close(queries)
+ c.logger.Error("[clickhouse] context canceled while normalizing",
+ slog.Any("error", errCtx.Err()),
+ slog.Any("cause", context.Cause(errCtx)))
+ return nil, context.Cause(errCtx)
}
}
+ close(queries)
+ if err := group.Wait(); err != nil {
+ return nil, err
+ }
- err = c.UpdateNormalizeBatchID(ctx, req.FlowJobName, req.SyncBatchID)
- if err != nil {
+ if err := c.UpdateNormalizeBatchID(ctx, req.FlowJobName, req.SyncBatchID); err != nil {
c.logger.Error("[clickhouse] error while updating normalize batch id", slog.Int64("BatchID", req.SyncBatchID), slog.Any("error", err))
return nil, err
}
@@ -462,7 +488,7 @@ func (c *ClickHouseConnector) getDistinctTableNamesInBatch(
rawTbl := c.getRawTableName(flowJobName)
q := fmt.Sprintf(
- `SELECT DISTINCT _peerdb_destination_table_name FROM %s WHERE _peerdb_batch_id > %d AND _peerdb_batch_id <= %d`,
+ `SELECT DISTINCT _peerdb_destination_table_name FROM %s WHERE _peerdb_batch_id>%d AND _peerdb_batch_id<=%d`,
rawTbl, normalizeBatchID, syncBatchID)
rows, err := c.query(ctx, q)
@@ -510,8 +536,7 @@ func (c *ClickHouseConnector) copyAvroStagesToDestination(
ctx context.Context, flowJobName string, normBatchID, syncBatchID int64,
) error {
for s := normBatchID + 1; s <= syncBatchID; s++ {
- err := c.copyAvroStageToDestination(ctx, flowJobName, s)
- if err != nil {
+ if err := c.copyAvroStageToDestination(ctx, flowJobName, s); err != nil {
return fmt.Errorf("failed to copy avro stage to destination: %w", err)
}
}
diff --git a/flow/connectors/clickhouse/qrep_avro_sync.go b/flow/connectors/clickhouse/qrep_avro_sync.go
index f8277e3aad..61450dd55c 100644
--- a/flow/connectors/clickhouse/qrep_avro_sync.go
+++ b/flow/connectors/clickhouse/qrep_avro_sync.go
@@ -60,6 +60,7 @@ func (s *ClickHouseAvroSyncMethod) CopyStageToDestination(ctx context.Context, a
func (s *ClickHouseAvroSyncMethod) SyncRecords(
ctx context.Context,
+ env map[string]string,
stream *model.QRecordStream,
flowJobName string,
syncBatchID int64,
@@ -70,13 +71,13 @@ func (s *ClickHouseAvroSyncMethod) SyncRecords(
s.logger.Info("sync function called and schema acquired",
slog.String("dstTable", dstTableName))
- avroSchema, err := s.getAvroSchema(dstTableName, schema)
+ avroSchema, err := s.getAvroSchema(ctx, env, dstTableName, schema)
if err != nil {
return 0, err
}
batchIdentifierForFile := fmt.Sprintf("%s_%d", shared.RandomString(16), syncBatchID)
- avroFile, err := s.writeToAvroFile(ctx, stream, avroSchema, batchIdentifierForFile, flowJobName)
+ avroFile, err := s.writeToAvroFile(ctx, env, stream, avroSchema, batchIdentifierForFile, flowJobName)
if err != nil {
return 0, err
}
@@ -105,12 +106,12 @@ func (s *ClickHouseAvroSyncMethod) SyncQRepRecords(
stagingPath := s.credsProvider.BucketPath
startTime := time.Now()
- avroSchema, err := s.getAvroSchema(dstTableName, stream.Schema())
+ avroSchema, err := s.getAvroSchema(ctx, config.Env, dstTableName, stream.Schema())
if err != nil {
return 0, err
}
- avroFile, err := s.writeToAvroFile(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName)
+ avroFile, err := s.writeToAvroFile(ctx, config.Env, stream, avroSchema, partition.PartitionId, config.FlowJobName)
if err != nil {
return 0, err
}
@@ -164,10 +165,12 @@ func (s *ClickHouseAvroSyncMethod) SyncQRepRecords(
}
func (s *ClickHouseAvroSyncMethod) getAvroSchema(
+ ctx context.Context,
+ env map[string]string,
dstTableName string,
schema qvalue.QRecordSchema,
) (*model.QRecordAvroSchemaDefinition, error) {
- avroSchema, err := model.GetAvroSchemaDefinition(dstTableName, schema, protos.DBType_CLICKHOUSE)
+ avroSchema, err := model.GetAvroSchemaDefinition(ctx, env, dstTableName, schema, protos.DBType_CLICKHOUSE)
if err != nil {
return nil, fmt.Errorf("failed to define Avro schema: %w", err)
}
@@ -176,6 +179,7 @@ func (s *ClickHouseAvroSyncMethod) getAvroSchema(
func (s *ClickHouseAvroSyncMethod) writeToAvroFile(
ctx context.Context,
+ env map[string]string,
stream *model.QRecordStream,
avroSchema *model.QRecordAvroSchemaDefinition,
identifierForFile string,
@@ -190,7 +194,7 @@ func (s *ClickHouseAvroSyncMethod) writeToAvroFile(
s3AvroFileKey := fmt.Sprintf("%s/%s/%s.avro.zst", s3o.Prefix, flowJobName, identifierForFile)
s3AvroFileKey = strings.Trim(s3AvroFileKey, "/")
- avroFile, err := ocfWriter.WriteRecordsToS3(ctx, s3o.Bucket, s3AvroFileKey, s.credsProvider.Provider)
+ avroFile, err := ocfWriter.WriteRecordsToS3(ctx, env, s3o.Bucket, s3AvroFileKey, s.credsProvider.Provider)
if err != nil {
return nil, fmt.Errorf("failed to write records to S3: %w", err)
}
diff --git a/flow/connectors/core.go b/flow/connectors/core.go
index 8a6bbbc0e2..0991a50978 100644
--- a/flow/connectors/core.go
+++ b/flow/connectors/core.go
@@ -23,7 +23,7 @@ import (
connsqlserver "github.com/PeerDB-io/peer-flow/connectors/sqlserver"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
- "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_gauges"
+ "github.com/PeerDB-io/peer-flow/otel_metrics"
"github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
)
@@ -85,7 +85,7 @@ type CDCPullConnectorCore interface {
alerter *alerting.Alerter,
catalogPool *pgxpool.Pool,
alertKeys *alerting.AlertKeys,
- slotMetricGauges peerdb_gauges.SlotMetricGauges,
+ slotMetricGauges otel_metrics.SlotMetricGauges,
) error
// GetSlotInfo returns the WAL (or equivalent) info of a slot for the connector.
@@ -102,7 +102,12 @@ type CDCPullConnector interface {
CDCPullConnectorCore
// This method should be idempotent, and should be able to be called multiple times with the same request.
- PullRecords(ctx context.Context, catalogPool *pgxpool.Pool, req *model.PullRecordsRequest[model.RecordItems]) error
+ PullRecords(
+ ctx context.Context,
+ catalogPool *pgxpool.Pool,
+ otelManager *otel_metrics.OtelManager,
+ req *model.PullRecordsRequest[model.RecordItems],
+ ) error
}
type CDCPullPgConnector interface {
@@ -110,7 +115,12 @@ type CDCPullPgConnector interface {
// This method should be idempotent, and should be able to be called multiple times with the same request.
// It's signature, aside from type parameter, should match CDCPullConnector.PullRecords.
- PullPg(ctx context.Context, catalogPool *pgxpool.Pool, req *model.PullRecordsRequest[model.PgItems]) error
+ PullPg(
+ ctx context.Context,
+ catalogPool *pgxpool.Pool,
+ otelManager *otel_metrics.OtelManager,
+ req *model.PullRecordsRequest[model.PgItems],
+ ) error
}
type NormalizedTablesConnector interface {
@@ -163,7 +173,7 @@ type CDCSyncConnectorCore interface {
// ReplayTableSchemaDelta changes a destination table to match the schema at source
// This could involve adding or dropping multiple columns.
// Connectors which are non-normalizing should implement this as a nop.
- ReplayTableSchemaDeltas(ctx context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error
+ ReplayTableSchemaDeltas(ctx context.Context, env map[string]string, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error
}
type CDCSyncConnector interface {
@@ -453,8 +463,6 @@ var (
_ CDCSyncConnector = &connclickhouse.ClickHouseConnector{}
_ CDCSyncConnector = &connelasticsearch.ElasticsearchConnector{}
- _ CDCSyncPgConnector = &connpostgres.PostgresConnector{}
-
_ CDCNormalizeConnector = &connpostgres.PostgresConnector{}
_ CDCNormalizeConnector = &connbigquery.BigQueryConnector{}
_ CDCNormalizeConnector = &connsnowflake.SnowflakeConnector{}
diff --git a/flow/connectors/elasticsearch/elasticsearch.go b/flow/connectors/elasticsearch/elasticsearch.go
index e675168051..30279fd74e 100644
--- a/flow/connectors/elasticsearch/elasticsearch.go
+++ b/flow/connectors/elasticsearch/elasticsearch.go
@@ -95,7 +95,7 @@ func (esc *ElasticsearchConnector) CreateRawTable(ctx context.Context,
}
// we handle schema changes by not handling them since no mapping is being enforced right now
-func (esc *ElasticsearchConnector) ReplayTableSchemaDeltas(ctx context.Context,
+func (esc *ElasticsearchConnector) ReplayTableSchemaDeltas(ctx context.Context, env map[string]string,
flowJobName string, schemaDeltas []*protos.TableSchemaDelta,
) error {
return nil
diff --git a/flow/connectors/eventhub/eventhub.go b/flow/connectors/eventhub/eventhub.go
index 01982bf713..0f175233ef 100644
--- a/flow/connectors/eventhub/eventhub.go
+++ b/flow/connectors/eventhub/eventhub.go
@@ -380,7 +380,9 @@ func (c *EventHubConnector) CreateRawTable(ctx context.Context, req *protos.Crea
}, nil
}
-func (c *EventHubConnector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error {
+func (c *EventHubConnector) ReplayTableSchemaDeltas(_ context.Context, _ map[string]string,
+ flowJobName string, schemaDeltas []*protos.TableSchemaDelta,
+) error {
c.logger.Info("ReplayTableSchemaDeltas for event hub is a no-op")
return nil
}
diff --git a/flow/connectors/eventhub/hubmanager.go b/flow/connectors/eventhub/hubmanager.go
index 3e134968dc..5515eae880 100644
--- a/flow/connectors/eventhub/hubmanager.go
+++ b/flow/connectors/eventhub/hubmanager.go
@@ -14,8 +14,8 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventhub/armeventhub"
cmap "github.com/orcaman/concurrent-map/v2"
- "github.com/PeerDB-io/peer-flow/connectors/utils"
"github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
)
@@ -186,10 +186,10 @@ func (m *EventHubManager) EnsureEventHubExists(ctx context.Context, name ScopedE
func (m *EventHubManager) getEventHubMgmtClient(subID string) (*armeventhub.EventHubsClient, error) {
if subID == "" {
- envSubID, err := utils.GetAzureSubscriptionID()
- if err != nil {
- slog.Error("failed to get azure subscription id", slog.Any("error", err))
- return nil, err
+ envSubID := peerdbenv.GetEnvString("AZURE_SUBSCRIPTION_ID", "")
+ if envSubID == "" {
+ slog.Error("couldn't find AZURE_SUBSCRIPTION_ID in environment")
+ return nil, errors.New("couldn't find AZURE_SUBSCRIPTION_ID in environment")
}
subID = envSubID
}
diff --git a/flow/connectors/kafka/kafka.go b/flow/connectors/kafka/kafka.go
index ea0805b84b..ee78093fe6 100644
--- a/flow/connectors/kafka/kafka.go
+++ b/flow/connectors/kafka/kafka.go
@@ -149,7 +149,9 @@ func (c *KafkaConnector) SetupMetadataTables(_ context.Context) error {
return nil
}
-func (c *KafkaConnector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error {
+func (c *KafkaConnector) ReplayTableSchemaDeltas(_ context.Context, _ map[string]string,
+ flowJobName string, schemaDeltas []*protos.TableSchemaDelta,
+) error {
return nil
}
diff --git a/flow/connectors/postgres/cdc.go b/flow/connectors/postgres/cdc.go
index a355cfa00e..27ae89904c 100644
--- a/flow/connectors/postgres/cdc.go
+++ b/flow/connectors/postgres/cdc.go
@@ -14,6 +14,8 @@ import (
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/lib/pq/oid"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"go.temporal.io/sdk/activity"
connmetadata "github.com/PeerDB-io/peer-flow/connectors/external_metadata"
@@ -22,6 +24,7 @@ import (
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/model/qvalue"
+ "github.com/PeerDB-io/peer-flow/otel_metrics"
"github.com/PeerDB-io/peer-flow/shared"
)
@@ -41,12 +44,14 @@ type PostgresCDCSource struct {
// for storing schema delta audit logs to catalog
catalogPool *pgxpool.Pool
+ otelManager *otel_metrics.OtelManager
hushWarnUnhandledMessageType map[pglogrepl.MessageType]struct{}
flowJobName string
}
type PostgresCDCConfig struct {
CatalogPool *pgxpool.Pool
+ OtelManager *otel_metrics.OtelManager
SrcTableIDNameMapping map[uint32]string
TableNameMapping map[string]model.NameAndExclude
TableNameSchemaMapping map[string]*protos.TableSchema
@@ -67,10 +72,11 @@ func (c *PostgresConnector) NewPostgresCDCSource(cdcConfig *PostgresCDCConfig) *
relationMessageMapping: cdcConfig.RelationMessageMapping,
slot: cdcConfig.Slot,
publication: cdcConfig.Publication,
- childToParentRelIDMapping: cdcConfig.ChildToParentRelIDMap,
typeMap: pgtype.NewMap(),
commitLock: nil,
+ childToParentRelIDMapping: cdcConfig.ChildToParentRelIDMap,
catalogPool: cdcConfig.CatalogPool,
+ otelManager: cdcConfig.OtelManager,
flowJobName: cdcConfig.FlowJobName,
hushWarnUnhandledMessageType: make(map[pglogrepl.MessageType]struct{}),
}
@@ -85,21 +91,18 @@ func GetChildToParentRelIDMap(ctx context.Context, conn *pgx.Conn) (map[uint32]u
WHERE parent.relkind='p';
`
- rows, err := conn.Query(ctx, query, pgx.QueryExecModeSimpleProtocol)
+ rows, err := conn.Query(ctx, query)
if err != nil {
return nil, fmt.Errorf("error querying for child to parent relid map: %w", err)
}
- defer rows.Close()
childToParentRelIDMap := make(map[uint32]uint32)
- var parentRelID pgtype.Uint32
- var childRelID pgtype.Uint32
- for rows.Next() {
- err := rows.Scan(&parentRelID, &childRelID)
- if err != nil {
- return nil, fmt.Errorf("error scanning child to parent relid map: %w", err)
- }
+ var parentRelID, childRelID pgtype.Uint32
+ if _, err := pgx.ForEachRow(rows, []any{&parentRelID, &childRelID}, func() error {
childToParentRelIDMap[childRelID.Uint32] = parentRelID.Uint32
+ return nil
+ }); err != nil {
+ return nil, fmt.Errorf("error iterating over child to parent relid map: %w", err)
}
return childToParentRelIDMap, nil
@@ -114,6 +117,7 @@ type replProcessor[Items model.Items] interface {
p *PostgresCDCSource,
tuple *pglogrepl.TupleDataColumn,
col *pglogrepl.RelationMessageColumn,
+ customTypeMapping map[uint32]string,
) error
}
@@ -128,6 +132,7 @@ func (pgProcessor) Process(
p *PostgresCDCSource,
tuple *pglogrepl.TupleDataColumn,
col *pglogrepl.RelationMessageColumn,
+ customTypeMapping map[uint32]string,
) error {
switch tuple.DataType {
case 'n': // null
@@ -158,13 +163,14 @@ func (qProcessor) Process(
p *PostgresCDCSource,
tuple *pglogrepl.TupleDataColumn,
col *pglogrepl.RelationMessageColumn,
+ customTypeMapping map[uint32]string,
) error {
switch tuple.DataType {
case 'n': // null
items.AddColumn(col.Name, qvalue.QValueNull(qvalue.QValueKindInvalid))
case 't': // text
// bytea also appears here as a hex
- data, err := p.decodeColumnData(tuple.Data, col.DataType, pgtype.TextFormatCode)
+ data, err := p.decodeColumnData(tuple.Data, col.DataType, pgtype.TextFormatCode, customTypeMapping)
if err != nil {
p.logger.Error("error decoding text column data", slog.Any("error", err),
slog.String("columnName", col.Name), slog.Int64("dataType", int64(col.DataType)))
@@ -172,7 +178,7 @@ func (qProcessor) Process(
}
items.AddColumn(col.Name, data)
case 'b': // binary
- data, err := p.decodeColumnData(tuple.Data, col.DataType, pgtype.BinaryFormatCode)
+ data, err := p.decodeColumnData(tuple.Data, col.DataType, pgtype.BinaryFormatCode, customTypeMapping)
if err != nil {
return fmt.Errorf("error decoding binary column data: %w", err)
}
@@ -189,6 +195,7 @@ func processTuple[Items model.Items](
tuple *pglogrepl.TupleData,
rel *pglogrepl.RelationMessage,
exclude map[string]struct{},
+ customTypeMapping map[uint32]string,
) (Items, map[string]struct{}, error) {
// if the tuple is nil, return an empty map
if tuple == nil {
@@ -208,7 +215,7 @@ func processTuple[Items model.Items](
unchangedToastColumns = make(map[string]struct{})
}
unchangedToastColumns[rcol.Name] = struct{}{}
- } else if err := processor.Process(items, p, tcol, rcol); err != nil {
+ } else if err := processor.Process(items, p, tcol, rcol, customTypeMapping); err != nil {
var none Items
return none, nil, err
}
@@ -216,7 +223,9 @@ func processTuple[Items model.Items](
return items, unchangedToastColumns, nil
}
-func (p *PostgresCDCSource) decodeColumnData(data []byte, dataType uint32, formatCode int16) (qvalue.QValue, error) {
+func (p *PostgresCDCSource) decodeColumnData(data []byte, dataType uint32,
+ formatCode int16, customTypeMapping map[uint32]string,
+) (qvalue.QValue, error) {
var parsedData any
var err error
if dt, ok := p.typeMap.TypeForOID(dataType); ok {
@@ -260,7 +269,7 @@ func (p *PostgresCDCSource) decodeColumnData(data []byte, dataType uint32, forma
return retVal, nil
}
- typeName, ok := p.customTypesMapping[dataType]
+ typeName, ok := customTypeMapping[dataType]
if ok {
customQKind := customTypeToQKind(typeName)
switch customQKind {
@@ -328,8 +337,7 @@ func PullCdcRecords[Items model.Items](
records.SignalAsEmpty()
}
logger.Info(fmt.Sprintf("[finished] PullRecords streamed %d records", cdcRecordsStorage.Len()))
- err := cdcRecordsStorage.Close()
- if err != nil {
+ if err := cdcRecordsStorage.Close(); err != nil {
logger.Warn("failed to clean up records storage", slog.Any("error", err))
}
}()
@@ -358,6 +366,16 @@ func PullCdcRecords[Items model.Items](
return nil
}
+ var fetchedBytesCounter metric.Int64Counter
+ if p.otelManager != nil {
+ var err error
+ fetchedBytesCounter, err = p.otelManager.GetOrInitInt64Counter(otel_metrics.BuildMetricName(otel_metrics.FetchedBytesCounterName),
+ metric.WithUnit("By"), metric.WithDescription("Bytes received of CopyData over replication slot"))
+ if err != nil {
+ return fmt.Errorf("could not get FetchedBytesCounter: %w", err)
+ }
+ }
+
pkmRequiresResponse := false
waitingForCommit := false
@@ -436,8 +454,7 @@ func PullCdcRecords[Items model.Items](
}()
cancel()
- ctxErr := ctx.Err()
- if ctxErr != nil {
+ if ctxErr := ctx.Err(); ctxErr != nil {
return fmt.Errorf("consumeStream preempted: %w", ctxErr)
}
@@ -460,6 +477,12 @@ func PullCdcRecords[Items model.Items](
continue
}
+ if fetchedBytesCounter != nil {
+ fetchedBytesCounter.Add(ctx, int64(len(msg.Data)), metric.WithAttributeSet(attribute.NewSet(
+ attribute.String(otel_metrics.FlowNameKey, req.FlowJobName),
+ )))
+ }
+
switch msg.Data[0] {
case pglogrepl.PrimaryKeepaliveMessageByteID:
pkm, err := pglogrepl.ParsePrimaryKeepaliveMessage(msg.Data[1:])
@@ -634,17 +657,21 @@ func processMessage[Items model.Items](
if err != nil {
return nil, fmt.Errorf("error parsing logical message: %w", err)
}
+ customTypeMapping, err := p.fetchCustomTypeMapping(ctx)
+ if err != nil {
+ return nil, err
+ }
switch msg := logicalMsg.(type) {
case *pglogrepl.BeginMessage:
logger.Debug("BeginMessage", slog.Any("FinalLSN", msg.FinalLSN), slog.Any("XID", msg.Xid))
p.commitLock = msg
case *pglogrepl.InsertMessage:
- return processInsertMessage(p, xld.WALStart, msg, processor)
+ return processInsertMessage(p, xld.WALStart, msg, processor, customTypeMapping)
case *pglogrepl.UpdateMessage:
- return processUpdateMessage(p, xld.WALStart, msg, processor)
+ return processUpdateMessage(p, xld.WALStart, msg, processor, customTypeMapping)
case *pglogrepl.DeleteMessage:
- return processDeleteMessage(p, xld.WALStart, msg, processor)
+ return processDeleteMessage(p, xld.WALStart, msg, processor, customTypeMapping)
case *pglogrepl.CommitMessage:
// for a commit message, update the last checkpoint id for the record batch.
logger.Debug("CommitMessage", slog.Any("CommitLSN", msg.CommitLSN), slog.Any("TransactionEndLSN", msg.TransactionEndLSN))
@@ -694,6 +721,7 @@ func processInsertMessage[Items model.Items](
lsn pglogrepl.LSN,
msg *pglogrepl.InsertMessage,
processor replProcessor[Items],
+ customTypeMapping map[uint32]string,
) (model.Record[Items], error) {
relID := p.getParentRelIDIfPartitioned(msg.RelationID)
@@ -710,7 +738,7 @@ func processInsertMessage[Items model.Items](
return nil, fmt.Errorf("unknown relation id: %d", relID)
}
- items, _, err := processTuple(processor, p, msg.Tuple, rel, p.tableNameMapping[tableName].Exclude)
+ items, _, err := processTuple(processor, p, msg.Tuple, rel, p.tableNameMapping[tableName].Exclude, customTypeMapping)
if err != nil {
return nil, fmt.Errorf("error converting tuple to map: %w", err)
}
@@ -729,6 +757,7 @@ func processUpdateMessage[Items model.Items](
lsn pglogrepl.LSN,
msg *pglogrepl.UpdateMessage,
processor replProcessor[Items],
+ customTypeMapping map[uint32]string,
) (model.Record[Items], error) {
relID := p.getParentRelIDIfPartitioned(msg.RelationID)
@@ -745,13 +774,14 @@ func processUpdateMessage[Items model.Items](
return nil, fmt.Errorf("unknown relation id: %d", relID)
}
- oldItems, _, err := processTuple(processor, p, msg.OldTuple, rel, p.tableNameMapping[tableName].Exclude)
+ oldItems, _, err := processTuple(processor, p, msg.OldTuple, rel,
+ p.tableNameMapping[tableName].Exclude, customTypeMapping)
if err != nil {
return nil, fmt.Errorf("error converting old tuple to map: %w", err)
}
newItems, unchangedToastColumns, err := processTuple(
- processor, p, msg.NewTuple, rel, p.tableNameMapping[tableName].Exclude)
+ processor, p, msg.NewTuple, rel, p.tableNameMapping[tableName].Exclude, customTypeMapping)
if err != nil {
return nil, fmt.Errorf("error converting new tuple to map: %w", err)
}
@@ -785,6 +815,7 @@ func processDeleteMessage[Items model.Items](
lsn pglogrepl.LSN,
msg *pglogrepl.DeleteMessage,
processor replProcessor[Items],
+ customTypeMapping map[uint32]string,
) (model.Record[Items], error) {
relID := p.getParentRelIDIfPartitioned(msg.RelationID)
@@ -801,7 +832,8 @@ func processDeleteMessage[Items model.Items](
return nil, fmt.Errorf("unknown relation id: %d", relID)
}
- items, _, err := processTuple(processor, p, msg.OldTuple, rel, p.tableNameMapping[tableName].Exclude)
+ items, _, err := processTuple(processor, p, msg.OldTuple, rel,
+ p.tableNameMapping[tableName].Exclude, customTypeMapping)
if err != nil {
return nil, fmt.Errorf("error converting tuple to map: %w", err)
}
@@ -844,6 +876,10 @@ func processRelationMessage[Items model.Items](
slog.Uint64("relId", uint64(currRel.RelationID)))
return nil, nil
}
+ customTypeMapping, err := p.fetchCustomTypeMapping(ctx)
+ if err != nil {
+ return nil, err
+ }
// retrieve current TableSchema for table changed, mapping uses dst table name as key, need to translate source name
currRelDstInfo, ok := p.tableNameMapping[currRelName]
@@ -867,7 +903,7 @@ func processRelationMessage[Items model.Items](
case protos.TypeSystem_Q:
qKind := p.postgresOIDToQValueKind(column.DataType)
if qKind == qvalue.QValueKindInvalid {
- typeName, ok := p.customTypesMapping[column.DataType]
+ typeName, ok := customTypeMapping[column.DataType]
if ok {
qKind = customTypeToQKind(typeName)
}
diff --git a/flow/connectors/postgres/client.go b/flow/connectors/postgres/client.go
index 1daabbf684..70b0d15d1d 100644
--- a/flow/connectors/postgres/client.go
+++ b/flow/connectors/postgres/client.go
@@ -550,7 +550,14 @@ func (c *PostgresConnector) jobMetadataExists(ctx context.Context, jobName strin
}
func (c *PostgresConnector) MajorVersion(ctx context.Context) (shared.PGVersion, error) {
- return shared.GetMajorVersion(ctx, c.conn)
+ if c.pgVersion == 0 {
+ pgVersion, err := shared.GetMajorVersion(ctx, c.conn)
+ if err != nil {
+ return 0, err
+ }
+ c.pgVersion = pgVersion
+ }
+ return c.pgVersion, nil
}
func (c *PostgresConnector) updateSyncMetadata(ctx context.Context, flowJobName string, lastCP int64, syncBatchID int64,
diff --git a/flow/connectors/postgres/postgres.go b/flow/connectors/postgres/postgres.go
index b3161161e1..8f49545fff 100644
--- a/flow/connectors/postgres/postgres.go
+++ b/flow/connectors/postgres/postgres.go
@@ -17,6 +17,7 @@ import (
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
"go.temporal.io/sdk/log"
"go.temporal.io/sdk/temporal"
@@ -27,7 +28,6 @@ import (
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/model/qvalue"
"github.com/PeerDB-io/peer-flow/otel_metrics"
- "github.com/PeerDB-io/peer-flow/otel_metrics/peerdb_gauges"
"github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
)
@@ -39,12 +39,13 @@ type PostgresConnector struct {
conn *pgx.Conn
replConn *pgx.Conn
replState *ReplState
- customTypesMapping map[uint32]string
+ customTypeMapping map[uint32]string
hushWarnOID map[uint32]struct{}
relationMessageMapping model.RelationMessageMapping
connStr string
metadataSchema string
replLock sync.Mutex
+ pgVersion shared.PGVersion
}
type ReplState struct {
@@ -87,33 +88,39 @@ func NewPostgresConnector(ctx context.Context, env map[string]string, pgConfig *
return nil, fmt.Errorf("failed to create connection: %w", err)
}
- customTypeMap, err := shared.GetCustomDataTypes(ctx, conn)
- if err != nil {
- logger.Error("failed to get custom type map", slog.Any("error", err))
- return nil, fmt.Errorf("failed to get custom type map: %w", err)
- }
-
metadataSchema := "_peerdb_internal"
if pgConfig.MetadataSchema != nil {
metadataSchema = *pgConfig.MetadataSchema
}
return &PostgresConnector{
- connStr: connectionString,
+ logger: logger,
config: pgConfig,
ssh: tunnel,
conn: conn,
replConn: nil,
replState: nil,
- replLock: sync.Mutex{},
- customTypesMapping: customTypeMap,
- metadataSchema: metadataSchema,
+ customTypeMapping: nil,
hushWarnOID: make(map[uint32]struct{}),
- logger: logger,
relationMessageMapping: make(model.RelationMessageMapping),
+ connStr: connectionString,
+ metadataSchema: metadataSchema,
+ replLock: sync.Mutex{},
+ pgVersion: 0,
}, nil
}
+func (c *PostgresConnector) fetchCustomTypeMapping(ctx context.Context) (map[uint32]string, error) {
+ if c.customTypeMapping == nil {
+ customTypeMapping, err := shared.GetCustomDataTypes(ctx, c.conn)
+ if err != nil {
+ return nil, err
+ }
+ c.customTypeMapping = customTypeMapping
+ }
+ return c.customTypeMapping, nil
+}
+
func (c *PostgresConnector) CreateReplConn(ctx context.Context) (*pgx.Conn, error) {
// create a separate connection pool for non-replication queries as replication connections cannot
// be used for extended query protocol, i.e. prepared statements
@@ -129,6 +136,7 @@ func (c *PostgresConnector) CreateReplConn(ctx context.Context) (*pgx.Conn, erro
replConfig.Config.RuntimeParams["replication"] = "database"
replConfig.Config.RuntimeParams["bytea_output"] = "hex"
replConfig.Config.RuntimeParams["intervalstyle"] = "postgres"
+ replConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol
conn, err := c.ssh.NewPostgresConnFromConfig(ctx, replConfig)
if err != nil {
@@ -168,6 +176,7 @@ func (c *PostgresConnector) MaybeStartReplication(
slotName string,
publicationName string,
lastOffset int64,
+ pgVersion shared.PGVersion,
) error {
if c.replState != nil && (c.replState.Offset != lastOffset ||
c.replState.Slot != slotName ||
@@ -180,7 +189,7 @@ func (c *PostgresConnector) MaybeStartReplication(
}
if c.replState == nil {
- replicationOpts, err := c.replicationOptions(ctx, publicationName)
+ replicationOpts, err := c.replicationOptions(publicationName, pgVersion)
if err != nil {
return fmt.Errorf("error getting replication options: %w", err)
}
@@ -210,7 +219,8 @@ func (c *PostgresConnector) MaybeStartReplication(
return nil
}
-func (c *PostgresConnector) replicationOptions(ctx context.Context, publicationName string) (pglogrepl.StartReplicationOptions, error) {
+func (c *PostgresConnector) replicationOptions(publicationName string, pgVersion shared.PGVersion,
+) (pglogrepl.StartReplicationOptions, error) {
pluginArguments := append(make([]string, 0, 3), "proto_version '1'")
if publicationName != "" {
@@ -220,10 +230,7 @@ func (c *PostgresConnector) replicationOptions(ctx context.Context, publicationN
return pglogrepl.StartReplicationOptions{}, errors.New("publication name is not set")
}
- pgversion, err := c.MajorVersion(ctx)
- if err != nil {
- return pglogrepl.StartReplicationOptions{}, err
- } else if pgversion >= shared.POSTGRES_14 {
+ if pgVersion >= shared.POSTGRES_14 {
pluginArguments = append(pluginArguments, "messages 'true'")
}
@@ -322,17 +329,19 @@ func (c *PostgresConnector) SetLastOffset(ctx context.Context, jobName string, l
func (c *PostgresConnector) PullRecords(
ctx context.Context,
catalogPool *pgxpool.Pool,
+ otelManager *otel_metrics.OtelManager,
req *model.PullRecordsRequest[model.RecordItems],
) error {
- return pullCore(ctx, c, catalogPool, req, qProcessor{})
+ return pullCore(ctx, c, catalogPool, otelManager, req, qProcessor{})
}
func (c *PostgresConnector) PullPg(
ctx context.Context,
catalogPool *pgxpool.Pool,
+ otelManager *otel_metrics.OtelManager,
req *model.PullRecordsRequest[model.PgItems],
) error {
- return pullCore(ctx, c, catalogPool, req, pgProcessor{})
+ return pullCore(ctx, c, catalogPool, otelManager, req, pgProcessor{})
}
// PullRecords pulls records from the source.
@@ -340,6 +349,7 @@ func pullCore[Items model.Items](
ctx context.Context,
c *PostgresConnector,
catalogPool *pgxpool.Pool,
+ otelManager *otel_metrics.OtelManager,
req *model.PullRecordsRequest[Items],
processor replProcessor[Items],
) error {
@@ -380,12 +390,21 @@ func pullCore[Items model.Items](
c.logger.Info("PullRecords: performed checks for slot and publication")
- childToParentRelIDMap, err := GetChildToParentRelIDMap(ctx, c.conn)
+ // cached, since this connector is reused
+ pgVersion, err := c.MajorVersion(ctx)
if err != nil {
- return fmt.Errorf("error getting child to parent relid map: %w", err)
+ return err
+ }
+ var childToParentRelIDMap map[uint32]uint32
+ // only initialize the map if needed, escape hatch because custom publications may not have the right setting
+ if req.OverridePublicationName != "" || pgVersion < shared.POSTGRES_13 {
+ childToParentRelIDMap, err = GetChildToParentRelIDMap(ctx, c.conn)
+ if err != nil {
+ return fmt.Errorf("error getting child to parent relid map: %w", err)
+ }
}
- if err := c.MaybeStartReplication(ctx, slotName, publicationName, req.LastOffset); err != nil {
+ if err := c.MaybeStartReplication(ctx, slotName, publicationName, req.LastOffset, pgVersion); err != nil {
// in case of Aurora error ERROR: replication slots cannot be used on RO (Read Only) node (SQLSTATE 55000)
if shared.IsSQLStateError(err, pgerrcode.ObjectNotInPrerequisiteState) &&
strings.Contains(err.Error(), "replication slots cannot be used on RO (Read Only) node") {
@@ -396,15 +415,16 @@ func pullCore[Items model.Items](
}
cdc := c.NewPostgresCDCSource(&PostgresCDCConfig{
+ CatalogPool: catalogPool,
+ OtelManager: otelManager,
SrcTableIDNameMapping: req.SrcTableIDNameMapping,
- Slot: slotName,
- Publication: publicationName,
TableNameMapping: req.TableNameMapping,
TableNameSchemaMapping: req.TableNameSchemaMapping,
ChildToParentRelIDMap: childToParentRelIDMap,
- CatalogPool: catalogPool,
- FlowJobName: req.FlowJobName,
RelationMessageMapping: c.relationMessageMapping,
+ FlowJobName: req.FlowJobName,
+ Slot: slotName,
+ Publication: publicationName,
})
if err := PullCdcRecords(ctx, cdc, req, processor, &c.replLock); err != nil {
@@ -418,8 +438,7 @@ func pullCore[Items model.Items](
return fmt.Errorf("failed to get current LSN: %w", err)
}
- err = monitoring.UpdateLatestLSNAtSourceForCDCFlow(ctx, catalogPool, req.FlowJobName, int64(latestLSN))
- if err != nil {
+ if err := monitoring.UpdateLatestLSNAtSourceForCDCFlow(ctx, catalogPool, req.FlowJobName, int64(latestLSN)); err != nil {
c.logger.Error("error updating latest LSN at source for CDC flow", slog.Any("error", err))
return fmt.Errorf("failed to update latest LSN at source for CDC flow: %w", err)
}
@@ -573,7 +592,7 @@ func syncRecordsCore[Items model.Items](
return nil, err
}
- err = c.ReplayTableSchemaDeltas(ctx, req.FlowJobName, req.Records.SchemaDeltas)
+ err = c.ReplayTableSchemaDeltas(ctx, req.Env, req.FlowJobName, req.Records.SchemaDeltas)
if err != nil {
return nil, fmt.Errorf("failed to sync schema changes: %w", err)
}
@@ -766,6 +785,10 @@ func (c *PostgresConnector) getTableSchemaForTable(
if err != nil {
return nil, err
}
+ customTypeMapping, err := c.fetchCustomTypeMapping(ctx)
+ if err != nil {
+ return nil, err
+ }
relID, err := c.getRelIDForTable(ctx, schemaTable)
if err != nil {
@@ -811,7 +834,7 @@ func (c *PostgresConnector) getTableSchemaForTable(
case protos.TypeSystem_PG:
colType = c.postgresOIDToName(fieldDescription.DataTypeOID)
if colType == "" {
- typeName, ok := c.customTypesMapping[fieldDescription.DataTypeOID]
+ typeName, ok := customTypeMapping[fieldDescription.DataTypeOID]
if !ok {
return nil, fmt.Errorf("error getting type name for %d", fieldDescription.DataTypeOID)
}
@@ -820,7 +843,7 @@ func (c *PostgresConnector) getTableSchemaForTable(
case protos.TypeSystem_Q:
qColType := c.postgresOIDToQValueKind(fieldDescription.DataTypeOID)
if qColType == qvalue.QValueKindInvalid {
- typeName, ok := c.customTypesMapping[fieldDescription.DataTypeOID]
+ typeName, ok := customTypeMapping[fieldDescription.DataTypeOID]
if ok {
qColType = customTypeToQKind(typeName)
} else {
@@ -891,15 +914,17 @@ func (c *PostgresConnector) SetupNormalizedTable(
if tableAlreadyExists {
c.logger.Info("[postgres] table already exists, skipping",
slog.String("table", tableIdentifier))
- if config.IsResync {
- err := c.ExecuteCommand(ctx, fmt.Sprintf(dropTableIfExistsSQL,
- QuoteIdentifier(parsedNormalizedTable.Schema),
- QuoteIdentifier(parsedNormalizedTable.Table)))
- if err != nil {
- return false, fmt.Errorf("error while dropping _resync table: %w", err)
- }
+ if !config.IsResync {
+ return true, nil
}
- return true, nil
+
+ err := c.ExecuteCommand(ctx, fmt.Sprintf(dropTableIfExistsSQL,
+ QuoteIdentifier(parsedNormalizedTable.Schema),
+ QuoteIdentifier(parsedNormalizedTable.Table)))
+ if err != nil {
+ return false, fmt.Errorf("error while dropping _resync table: %w", err)
+ }
+ c.logger.Info("[postgres] dropped resync table for resync", slog.String("resyncTable", parsedNormalizedTable.String()))
}
// convert the column names and types to Postgres types
@@ -916,6 +941,7 @@ func (c *PostgresConnector) SetupNormalizedTable(
// This could involve adding or dropping multiple columns.
func (c *PostgresConnector) ReplayTableSchemaDeltas(
ctx context.Context,
+ _ map[string]string,
flowJobName string,
schemaDeltas []*protos.TableSchemaDelta,
) error {
@@ -1165,6 +1191,7 @@ func (c *PostgresConnector) SyncFlowCleanup(ctx context.Context, jobName string)
if err := syncFlowCleanupTx.Commit(ctx); err != nil {
return fmt.Errorf("unable to commit transaction for sync flow cleanup: %w", err)
}
+
return nil
}
@@ -1173,7 +1200,7 @@ func (c *PostgresConnector) HandleSlotInfo(
alerter *alerting.Alerter,
catalogPool *pgxpool.Pool,
alertKeys *alerting.AlertKeys,
- slotMetricGauges peerdb_gauges.SlotMetricGauges,
+ slotMetricGauges otel_metrics.SlotMetricGauges,
) error {
logger := shared.LoggerFromCtx(ctx)
@@ -1191,11 +1218,16 @@ func (c *PostgresConnector) HandleSlotInfo(
logger.Info(fmt.Sprintf("Checking %s lag for %s", alertKeys.SlotName, alertKeys.PeerName),
slog.Float64("LagInMB", float64(slotInfo[0].LagInMb)))
alerter.AlertIfSlotLag(ctx, alertKeys, slotInfo[0])
- slotMetricGauges.SlotLagGauge.Set(float64(slotInfo[0].LagInMb), attribute.NewSet(
- attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
- attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
- attribute.String(otel_metrics.SlotNameKey, alertKeys.SlotName),
- attribute.String(otel_metrics.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID())))
+
+ if slotMetricGauges.SlotLagGauge != nil {
+ slotMetricGauges.SlotLagGauge.Record(ctx, float64(slotInfo[0].LagInMb), metric.WithAttributeSet(attribute.NewSet(
+ attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
+ attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
+ attribute.String(otel_metrics.SlotNameKey, alertKeys.SlotName),
+ )))
+ } else {
+ logger.Warn("warning: slotMetricGauges.SlotLagGauge is nil")
+ }
// Also handles alerts for PeerDB user connections exceeding a given limit here
res, err := getOpenConnectionsForUser(ctx, c.conn, c.config.User)
@@ -1204,21 +1236,31 @@ func (c *PostgresConnector) HandleSlotInfo(
return err
}
alerter.AlertIfOpenConnections(ctx, alertKeys, res)
- slotMetricGauges.OpenConnectionsGauge.Set(res.CurrentOpenConnections, attribute.NewSet(
- attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
- attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
- attribute.String(otel_metrics.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID())))
+ if slotMetricGauges.OpenConnectionsGauge != nil {
+ slotMetricGauges.OpenConnectionsGauge.Record(ctx, res.CurrentOpenConnections, metric.WithAttributeSet(attribute.NewSet(
+ attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
+ attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
+ )))
+ } else {
+ logger.Warn("warning: slotMetricGauges.OpenConnectionsGauge is nil")
+ }
replicationRes, err := getOpenReplicationConnectionsForUser(ctx, c.conn, c.config.User)
if err != nil {
logger.Warn("warning: failed to get current open replication connections", "error", err)
return err
}
- slotMetricGauges.OpenReplicationConnectionsGauge.Set(replicationRes.CurrentOpenConnections, attribute.NewSet(
- attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
- attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
- attribute.String(otel_metrics.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID())))
+ if slotMetricGauges.OpenReplicationConnectionsGauge != nil {
+ slotMetricGauges.OpenReplicationConnectionsGauge.Record(ctx, replicationRes.CurrentOpenConnections,
+ metric.WithAttributeSet(attribute.NewSet(
+ attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
+ attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
+ )),
+ )
+ } else {
+ logger.Warn("warning: slotMetricGauges.OpenReplicationConnectionsGauge is nil")
+ }
var intervalSinceLastNormalize *time.Duration
if err := alerter.CatalogPool.QueryRow(
@@ -1232,10 +1274,16 @@ func (c *PostgresConnector) HandleSlotInfo(
return nil
}
if intervalSinceLastNormalize != nil {
- slotMetricGauges.IntervalSinceLastNormalizeGauge.Set(intervalSinceLastNormalize.Seconds(), attribute.NewSet(
- attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
- attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
- attribute.String(otel_metrics.DeploymentUidKey, peerdbenv.PeerDBDeploymentUID())))
+ if slotMetricGauges.IntervalSinceLastNormalizeGauge != nil {
+ slotMetricGauges.IntervalSinceLastNormalizeGauge.Record(ctx, intervalSinceLastNormalize.Seconds(),
+ metric.WithAttributeSet(attribute.NewSet(
+ attribute.String(otel_metrics.FlowNameKey, alertKeys.FlowName),
+ attribute.String(otel_metrics.PeerNameKey, alertKeys.PeerName),
+ )),
+ )
+ } else {
+ logger.Warn("warning: slotMetricGauges.IntervalSinceLastNormalizeGauge is nil")
+ }
alerter.AlertIfTooLongSinceLastNormalize(ctx, alertKeys, *intervalSinceLastNormalize)
}
@@ -1437,7 +1485,7 @@ func (c *PostgresConnector) RenameTables(
}
// rename the src table to dst
- _, err = c.execWithLoggingTx(ctx, fmt.Sprintf("ALTER TABLE %s RENAME TO %s", src, dstTable.Table), renameTablesTx)
+ _, err = c.execWithLoggingTx(ctx, fmt.Sprintf("ALTER TABLE %s RENAME TO %s", src, QuoteIdentifier(dstTable.Table)), renameTablesTx)
if err != nil {
return nil, fmt.Errorf("unable to rename table %s to %s: %w", src, dst, err)
}
diff --git a/flow/connectors/postgres/postgres_schema_delta_test.go b/flow/connectors/postgres/postgres_schema_delta_test.go
index 946b20eb3e..0b6668a5a2 100644
--- a/flow/connectors/postgres/postgres_schema_delta_test.go
+++ b/flow/connectors/postgres/postgres_schema_delta_test.go
@@ -58,7 +58,7 @@ func (s PostgresSchemaDeltaTestSuite) TestSimpleAddColumn() {
fmt.Sprintf("CREATE TABLE %s(id INT PRIMARY KEY)", tableName))
require.NoError(s.t, err)
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: []*protos.FieldDescription{
@@ -113,7 +113,7 @@ func (s PostgresSchemaDeltaTestSuite) TestAddAllColumnTypes() {
}
}
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: addedColumns,
@@ -144,7 +144,7 @@ func (s PostgresSchemaDeltaTestSuite) TestAddTrickyColumnNames() {
}
}
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: addedColumns,
@@ -175,7 +175,7 @@ func (s PostgresSchemaDeltaTestSuite) TestAddDropWhitespaceColumnNames() {
}
}
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: addedColumns,
diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go
index b393a46913..1cd2cd5952 100644
--- a/flow/connectors/postgres/qrep.go
+++ b/flow/connectors/postgres/qrep.go
@@ -328,10 +328,15 @@ func corePullQRepRecords(
sink QRepPullSink,
) (int, error) {
partitionIdLog := slog.String(string(shared.PartitionIDKey), partition.PartitionId)
+
if partition.FullTablePartition {
c.logger.Info("pulling full table partition", partitionIdLog)
- executor := c.NewQRepQueryExecutorSnapshot(config.SnapshotName, config.FlowJobName, partition.PartitionId)
- _, err := executor.ExecuteQueryIntoSink(ctx, sink, config.Query)
+ executor, err := c.NewQRepQueryExecutorSnapshot(ctx, config.SnapshotName,
+ config.FlowJobName, partition.PartitionId)
+ if err != nil {
+ return 0, fmt.Errorf("failed to create query executor: %w", err)
+ }
+ _, err = executor.ExecuteQueryIntoSink(ctx, sink, config.Query)
return 0, err
}
c.logger.Info("Obtained ranges for partition for PullQRepStream", partitionIdLog)
@@ -369,7 +374,11 @@ func corePullQRepRecords(
return 0, err
}
- executor := c.NewQRepQueryExecutorSnapshot(config.SnapshotName, config.FlowJobName, partition.PartitionId)
+ executor, err := c.NewQRepQueryExecutorSnapshot(ctx, config.SnapshotName, config.FlowJobName,
+ partition.PartitionId)
+ if err != nil {
+ return 0, fmt.Errorf("failed to create query executor: %w", err)
+ }
numRecords, err := executor.ExecuteQueryIntoSink(ctx, sink, query, rangeStart, rangeEnd)
if err != nil {
@@ -669,7 +678,11 @@ func pullXminRecordStream(
queryArgs = []interface{}{strconv.FormatInt(partition.Range.Range.(*protos.PartitionRange_IntRange).IntRange.Start&0xffffffff, 10)}
}
- executor := c.NewQRepQueryExecutorSnapshot(config.SnapshotName, config.FlowJobName, partition.PartitionId)
+ executor, err := c.NewQRepQueryExecutorSnapshot(ctx, config.SnapshotName,
+ config.FlowJobName, partition.PartitionId)
+ if err != nil {
+ return 0, 0, fmt.Errorf("failed to create query executor: %w", err)
+ }
numRecords, currentSnapshotXmin, err := executor.ExecuteQueryIntoSinkGettingCurrentSnapshotXmin(
ctx,
diff --git a/flow/connectors/postgres/qrep_bench_test.go b/flow/connectors/postgres/qrep_bench_test.go
index d880343f43..777faf6e6f 100644
--- a/flow/connectors/postgres/qrep_bench_test.go
+++ b/flow/connectors/postgres/qrep_bench_test.go
@@ -4,6 +4,8 @@ import (
"context"
"testing"
+ "github.com/stretchr/testify/require"
+
"github.com/PeerDB-io/peer-flow/peerdbenv"
)
@@ -12,13 +14,12 @@ func BenchmarkQRepQueryExecutor(b *testing.B) {
ctx := context.Background()
connector, err := NewPostgresConnector(ctx, nil, peerdbenv.GetCatalogPostgresConfigFromEnv(ctx))
- if err != nil {
- b.Fatalf("failed to create connection: %v", err)
- }
+ require.NoError(b, err, "error while creating connector")
defer connector.Close()
// Create a new QRepQueryExecutor instance
- qe := connector.NewQRepQueryExecutor("test flow", "test part")
+ qe, err := connector.NewQRepQueryExecutor(ctx, "test flow", "test part")
+ require.NoError(b, err, "error while creating QRepQueryExecutor")
// Run the benchmark
b.ResetTimer()
@@ -28,8 +29,6 @@ func BenchmarkQRepQueryExecutor(b *testing.B) {
// Execute the query and process the rows
_, err := qe.ExecuteAndProcessQuery(ctx, query)
- if err != nil {
- b.Fatalf("failed to execute query: %v", err)
- }
+ require.NoError(b, err, "error while executing query")
}
}
diff --git a/flow/connectors/postgres/qrep_partition_test.go b/flow/connectors/postgres/qrep_partition_test.go
index 0249b75fc1..a81df27698 100644
--- a/flow/connectors/postgres/qrep_partition_test.go
+++ b/flow/connectors/postgres/qrep_partition_test.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
+ "math/rand/v2"
"testing"
"time"
@@ -84,11 +85,8 @@ func TestGetQRepPartitions(t *testing.T) {
}
defer conn.Close(context.Background())
- // Generate a random schema name
- rndUint, err := shared.RandomUInt64()
- if err != nil {
- t.Fatalf("Failed to generate random uint: %v", err)
- }
+ //nolint:gosec // Generate a random schema name, number has no cryptographic significance
+ rndUint := rand.Uint64()
schemaName := fmt.Sprintf("test_%d", rndUint)
// Create the schema
diff --git a/flow/connectors/postgres/qrep_query_executor.go b/flow/connectors/postgres/qrep_query_executor.go
index bdfa7038ba..339c54a633 100644
--- a/flow/connectors/postgres/qrep_query_executor.go
+++ b/flow/connectors/postgres/qrep_query_executor.go
@@ -18,24 +18,35 @@ import (
type QRepQueryExecutor struct {
*PostgresConnector
- logger log.Logger
- snapshot string
- flowJobName string
- partitionID string
+ logger log.Logger
+ customTypeMapping map[uint32]string
+ snapshot string
+ flowJobName string
+ partitionID string
}
-func (c *PostgresConnector) NewQRepQueryExecutor(flowJobName string, partitionID string) *QRepQueryExecutor {
- return c.NewQRepQueryExecutorSnapshot("", flowJobName, partitionID)
+func (c *PostgresConnector) NewQRepQueryExecutor(ctx context.Context,
+ flowJobName string, partitionID string,
+) (*QRepQueryExecutor, error) {
+ return c.NewQRepQueryExecutorSnapshot(ctx, "", flowJobName, partitionID)
}
-func (c *PostgresConnector) NewQRepQueryExecutorSnapshot(snapshot string, flowJobName string, partitionID string) *QRepQueryExecutor {
+func (c *PostgresConnector) NewQRepQueryExecutorSnapshot(ctx context.Context,
+ snapshot string, flowJobName string, partitionID string,
+) (*QRepQueryExecutor, error) {
+ customTypeMapping, err := c.fetchCustomTypeMapping(ctx)
+ if err != nil {
+ c.logger.Error("[pg_query_executor] failed to fetch custom type mapping", slog.Any("error", err))
+ return nil, fmt.Errorf("failed to fetch custom type mapping: %w", err)
+ }
return &QRepQueryExecutor{
PostgresConnector: c,
snapshot: snapshot,
flowJobName: flowJobName,
partitionID: partitionID,
logger: log.With(c.logger, slog.String(string(shared.PartitionIDKey), partitionID)),
- }
+ customTypeMapping: customTypeMapping,
+ }, nil
}
func (qe *QRepQueryExecutor) ExecuteQuery(ctx context.Context, query string, args ...interface{}) (pgx.Rows, error) {
@@ -67,7 +78,7 @@ func (qe *QRepQueryExecutor) fieldDescriptionsToSchema(fds []pgconn.FieldDescrip
cname := fd.Name
ctype := qe.postgresOIDToQValueKind(fd.DataTypeOID)
if ctype == qvalue.QValueKindInvalid {
- typeName, ok := qe.customTypesMapping[fd.DataTypeOID]
+ typeName, ok := qe.customTypeMapping[fd.DataTypeOID]
if ok {
ctype = customTypeToQKind(typeName)
} else {
@@ -98,6 +109,7 @@ func (qe *QRepQueryExecutor) fieldDescriptionsToSchema(fds []pgconn.FieldDescrip
}
func (qe *QRepQueryExecutor) ProcessRows(
+ ctx context.Context,
rows pgx.Rows,
fieldDescriptions []pgconn.FieldDescription,
) (*model.QRecordBatch, error) {
@@ -119,8 +131,9 @@ func (qe *QRepQueryExecutor) ProcessRows(
return nil, fmt.Errorf("row iteration failed: %w", err)
}
+ schema := qe.fieldDescriptionsToSchema(fieldDescriptions)
batch := &model.QRecordBatch{
- Schema: qe.fieldDescriptionsToSchema(fieldDescriptions),
+ Schema: schema,
Records: records,
}
@@ -186,7 +199,8 @@ func (qe *QRepQueryExecutor) processFetchedRows(
fieldDescriptions := rows.FieldDescriptions()
if !stream.IsSchemaSet() {
- stream.SetSchema(qe.fieldDescriptionsToSchema(fieldDescriptions))
+ schema := qe.fieldDescriptionsToSchema(fieldDescriptions)
+ stream.SetSchema(schema)
}
numRows, err := qe.processRowsStream(ctx, cursorName, stream, rows, fieldDescriptions)
@@ -198,8 +212,8 @@ func (qe *QRepQueryExecutor) processFetchedRows(
if err := rows.Err(); err != nil {
stream.Close(err)
qe.logger.Error("[pg_query_executor] row iteration failed",
- slog.String("query", query), slog.Any("error", rows.Err()))
- return 0, fmt.Errorf("[pg_query_executor] row iteration failed '%s': %w", query, rows.Err())
+ slog.String("query", query), slog.Any("error", err))
+ return 0, fmt.Errorf("[pg_query_executor] row iteration failed '%s': %w", query, err)
}
return numRows, nil
@@ -324,7 +338,7 @@ func (qe *QRepQueryExecutor) mapRowToQRecord(
for i, fd := range fds {
// Check if it's a custom type first
- typeName, ok := qe.customTypesMapping[fd.DataTypeOID]
+ typeName, ok := qe.customTypeMapping[fd.DataTypeOID]
if !ok {
tmp, err := qe.parseFieldFromPostgresOID(fd.DataTypeOID, values[i])
if err != nil {
diff --git a/flow/connectors/postgres/qrep_query_executor_test.go b/flow/connectors/postgres/qrep_query_executor_test.go
index d7932ba00e..f8f686c42f 100644
--- a/flow/connectors/postgres/qrep_query_executor_test.go
+++ b/flow/connectors/postgres/qrep_query_executor_test.go
@@ -1,7 +1,6 @@
package connpostgres
import (
- "bytes"
"context"
"fmt"
"testing"
@@ -10,6 +9,7 @@ import (
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/shopspring/decimal"
+ "github.com/stretchr/testify/require"
"github.com/PeerDB-io/peer-flow/peerdbenv"
)
@@ -19,18 +19,14 @@ func setupDB(t *testing.T) (*PostgresConnector, string) {
connector, err := NewPostgresConnector(context.Background(),
nil, peerdbenv.GetCatalogPostgresConfigFromEnv(context.Background()))
- if err != nil {
- t.Fatalf("unable to create connector: %v", err)
- }
+ require.NoError(t, err, "error while creating connector")
// Create unique schema name using current time
schemaName := fmt.Sprintf("schema_%d", time.Now().Unix())
// Create the schema
_, err = connector.conn.Exec(context.Background(), fmt.Sprintf("CREATE SCHEMA %s;", schemaName))
- if err != nil {
- t.Fatalf("unable to create schema: %v", err)
- }
+ require.NoError(t, err, "error while creating schema")
return connector, schemaName
}
@@ -39,9 +35,7 @@ func teardownDB(t *testing.T, conn *pgx.Conn, schemaName string) {
t.Helper()
_, err := conn.Exec(context.Background(), fmt.Sprintf("DROP SCHEMA %s CASCADE;", schemaName))
- if err != nil {
- t.Fatalf("error while dropping schema: %v", err)
- }
+ require.NoError(t, err, "error while dropping schema")
}
func TestExecuteAndProcessQuery(t *testing.T) {
@@ -53,31 +47,20 @@ func TestExecuteAndProcessQuery(t *testing.T) {
query := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s.test(id SERIAL PRIMARY KEY, data TEXT);", schemaName)
_, err := conn.Exec(ctx, query)
- if err != nil {
- t.Fatalf("error while creating test table: %v", err)
- }
+ require.NoError(t, err, "error while creating table")
query = fmt.Sprintf("INSERT INTO %s.test(data) VALUES('testdata');", schemaName)
_, err = conn.Exec(ctx, query)
- if err != nil {
- t.Fatalf("error while inserting into test table: %v", err)
- }
+ require.NoError(t, err, "error while inserting data")
- qe := connector.NewQRepQueryExecutor("test flow", "test part")
+ qe, err := connector.NewQRepQueryExecutor(ctx, "test flow", "test part")
+ require.NoError(t, err, "error while creating QRepQueryExecutor")
query = fmt.Sprintf("SELECT * FROM %s.test;", schemaName)
batch, err := qe.ExecuteAndProcessQuery(context.Background(), query)
- if err != nil {
- t.Fatalf("error while executing and processing query: %v", err)
- }
-
- if len(batch.Records) != 1 {
- t.Fatalf("expected 1 record, got %v", len(batch.Records))
- }
-
- if batch.Records[0][1].Value() != "testdata" {
- t.Fatalf("expected 'testdata', got %v", batch.Records[0][0].Value())
- }
+ require.NoError(t, err, "error while executing query")
+ require.Len(t, batch.Records, 1, "expected 1 record")
+ require.Equal(t, "testdata", batch.Records[0][1].Value(), "expected 'testdata'")
}
func TestAllDataTypes(t *testing.T) {
@@ -109,9 +92,7 @@ func TestAllDataTypes(t *testing.T) {
);`, schemaName)
_, err := conn.Exec(ctx, query)
- if err != nil {
- t.Fatalf("error while creating test table: %v", err)
- }
+ require.NoError(t, err, "error while creating table")
// Insert a row into the table
query = fmt.Sprintf(`
@@ -137,7 +118,7 @@ func TestAllDataTypes(t *testing.T) {
)`,
schemaName)
- savedTime := time.Now()
+ savedTime := time.Now().UTC()
savedUUID := uuid.New()
_, err = conn.Exec(
@@ -160,48 +141,34 @@ func TestAllDataTypes(t *testing.T) {
savedTime, // col_tz4
savedTime, // col_date
)
- if err != nil {
- t.Fatalf("error while inserting into test table: %v", err)
- }
+ require.NoError(t, err, "error while inserting into test table")
- qe := connector.NewQRepQueryExecutor("test flow", "test part")
+ qe, err := connector.NewQRepQueryExecutor(ctx, "test flow", "test part")
+ require.NoError(t, err, "error while creating QRepQueryExecutor")
// Select the row back out of the table
query = fmt.Sprintf("SELECT * FROM %s.test;", schemaName)
rows, err := qe.ExecuteQuery(context.Background(), query)
- if err != nil {
- t.Fatalf("error while executing query: %v", err)
- }
+ require.NoError(t, err, "error while executing query")
defer rows.Close()
// Use rows.FieldDescriptions() to get field descriptions
fieldDescriptions := rows.FieldDescriptions()
- batch, err := qe.ProcessRows(rows, fieldDescriptions)
- if err != nil {
- t.Fatalf("failed to process rows: %v", err)
- }
-
- if len(batch.Records) != 1 {
- t.Fatalf("expected 1 record, got %v", len(batch.Records))
- }
+ batch, err := qe.ProcessRows(ctx, rows, fieldDescriptions)
+ require.NoError(t, err, "error while processing rows")
+ require.Len(t, batch.Records, 1, "expected 1 record")
// Retrieve the results.
record := batch.Records[0]
expectedBool := true
- if record[0].Value().(bool) != expectedBool {
- t.Fatalf("expected %v, got %v", expectedBool, record[0].Value())
- }
+ require.Equal(t, expectedBool, record[0].Value(), "expected true")
expectedInt4 := int32(2)
- if record[1].Value().(int32) != expectedInt4 {
- t.Fatalf("expected %v, got %v", expectedInt4, record[1].Value())
- }
+ require.Equal(t, expectedInt4, record[1].Value(), "expected 2")
expectedInt8 := int64(3)
- if record[2].Value().(int64) != expectedInt8 {
- t.Fatalf("expected %v, got %v", expectedInt8, record[2].Value())
- }
+ require.Equal(t, expectedInt8, record[2].Value(), "expected 3")
expectedFloat4 := float32(1.1)
if record[3].Value().(float32) != expectedFloat4 {
@@ -214,28 +181,21 @@ func TestAllDataTypes(t *testing.T) {
}
expectedText := "text"
- if record[5].Value().(string) != expectedText {
- t.Fatalf("expected %v, got %v", expectedText, record[5].Value())
- }
+ require.Equal(t, expectedText, record[5].Value(), "expected 'text'")
expectedBytea := []byte("bytea")
- if !bytes.Equal(record[6].Value().([]byte), expectedBytea) {
- t.Fatalf("expected %v, got %v", expectedBytea, record[6].Value())
- }
+ require.Equal(t, expectedBytea, record[6].Value(), "expected 'bytea'")
expectedJSON := `{"key":"value"}`
- if record[7].Value().(string) != expectedJSON {
- t.Fatalf("expected %v, got %v", expectedJSON, record[7].Value())
- }
+ require.Equal(t, expectedJSON, record[7].Value(), "expected '{\"key\":\"value\"}'")
actualUUID := record[8].Value().([16]uint8)
- if !bytes.Equal(actualUUID[:], savedUUID[:]) {
- t.Fatalf("expected %v, got %v", savedUUID, actualUUID)
- }
+ require.Equal(t, savedUUID[:], actualUUID[:], "expected savedUUID: %v", savedUUID)
+ actualTime := record[9].Value().(time.Time)
+ require.Equal(t, savedTime.Truncate(time.Second),
+ actualTime.Truncate(time.Second), "expected savedTime: %v", savedTime)
expectedNumeric := "123.456"
actualNumeric := record[10].Value().(decimal.Decimal).String()
- if actualNumeric != expectedNumeric {
- t.Fatalf("expected %v, got %v", expectedNumeric, actualNumeric)
- }
+ require.Equal(t, expectedNumeric, actualNumeric, "expected 123.456")
}
diff --git a/flow/connectors/postgres/qvalue_convert.go b/flow/connectors/postgres/qvalue_convert.go
index d359212bdb..fe2489ed30 100644
--- a/flow/connectors/postgres/qvalue_convert.go
+++ b/flow/connectors/postgres/qvalue_convert.go
@@ -62,8 +62,10 @@ func (c *PostgresConnector) postgresOIDToQValueKind(recvOID uint32) qvalue.QValu
return qvalue.QValueKindString
case pgtype.ByteaOID:
return qvalue.QValueKindBytes
- case pgtype.JSONOID, pgtype.JSONBOID:
+ case pgtype.JSONOID:
return qvalue.QValueKindJSON
+ case pgtype.JSONBOID:
+ return qvalue.QValueKindJSONB
case pgtype.UUIDOID:
return qvalue.QValueKindUUID
case pgtype.TimeOID:
@@ -104,8 +106,14 @@ func (c *PostgresConnector) postgresOIDToQValueKind(recvOID uint32) qvalue.QValu
return qvalue.QValueKindArrayTimestampTZ
case pgtype.TextArrayOID, pgtype.VarcharArrayOID, pgtype.BPCharArrayOID:
return qvalue.QValueKindArrayString
+ case pgtype.JSONArrayOID:
+ return qvalue.QValueKindArrayJSON
+ case pgtype.JSONBArrayOID:
+ return qvalue.QValueKindArrayJSONB
case pgtype.IntervalOID:
return qvalue.QValueKindInterval
+ case pgtype.TstzrangeOID:
+ return qvalue.QValueKindTSTZRange
default:
typeName, ok := pgtype.NewMap().TypeForOID(recvOID)
if !ok {
@@ -161,6 +169,8 @@ func qValueKindToPostgresType(colTypeStr string) string {
return "BYTEA"
case qvalue.QValueKindJSON:
return "JSON"
+ case qvalue.QValueKindJSONB:
+ return "JSONB"
case qvalue.QValueKindHStore:
return "HSTORE"
case qvalue.QValueKindUUID:
@@ -203,6 +213,10 @@ func qValueKindToPostgresType(colTypeStr string) string {
return "BOOLEAN[]"
case qvalue.QValueKindArrayString:
return "TEXT[]"
+ case qvalue.QValueKindArrayJSON:
+ return "JSON[]"
+ case qvalue.QValueKindArrayJSONB:
+ return "JSONB[]"
case qvalue.QValueKindGeography:
return "GEOGRAPHY"
case qvalue.QValueKindGeometry:
@@ -214,12 +228,12 @@ func qValueKindToPostgresType(colTypeStr string) string {
}
}
-func parseJSON(value interface{}) (qvalue.QValue, error) {
+func parseJSON(value interface{}, isArray bool) (qvalue.QValue, error) {
jsonVal, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON: %w", err)
}
- return qvalue.QValueJSON{Val: string(jsonVal)}, nil
+ return qvalue.QValueJSON{Val: string(jsonVal), IsArray: isArray}, nil
}
func convertToArray[T any](kind qvalue.QValueKind, value interface{}) ([]T, error) {
@@ -277,6 +291,31 @@ func parseFieldFromQValueKind(qvalueKind qvalue.QValueKind, value interface{}) (
}
return qvalue.QValueString{Val: string(intervalJSON)}, nil
+ case qvalue.QValueKindTSTZRange:
+ tstzrangeObject := value.(pgtype.Range[interface{}])
+ lowerBoundType := tstzrangeObject.LowerType
+ upperBoundType := tstzrangeObject.UpperType
+ lowerTime, err := convertTimeRangeBound(tstzrangeObject.Lower)
+ if err != nil {
+ return nil, fmt.Errorf("[tstzrange]error for lower time bound: %w", err)
+ }
+
+ upperTime, err := convertTimeRangeBound(tstzrangeObject.Upper)
+ if err != nil {
+ return nil, fmt.Errorf("[tstzrange]error for upper time bound: %w", err)
+ }
+
+ lowerBracket := "["
+ if lowerBoundType == pgtype.Exclusive {
+ lowerBracket = "("
+ }
+ upperBracket := "]"
+ if upperBoundType == pgtype.Exclusive {
+ upperBracket = ")"
+ }
+ tstzrangeStr := fmt.Sprintf("%s%v,%v%s",
+ lowerBracket, lowerTime, upperTime, upperBracket)
+ return qvalue.QValueTSTZRange{Val: tstzrangeStr}, nil
case qvalue.QValueKindDate:
switch val := value.(type) {
case time.Time:
@@ -306,12 +345,18 @@ func parseFieldFromQValueKind(qvalueKind qvalue.QValueKind, value interface{}) (
case qvalue.QValueKindBoolean:
boolVal := value.(bool)
return qvalue.QValueBoolean{Val: boolVal}, nil
- case qvalue.QValueKindJSON:
- tmp, err := parseJSON(value)
+ case qvalue.QValueKindJSON, qvalue.QValueKindJSONB:
+ tmp, err := parseJSON(value, false)
if err != nil {
return nil, fmt.Errorf("failed to parse JSON: %w", err)
}
return tmp, nil
+ case qvalue.QValueKindArrayJSON, qvalue.QValueKindArrayJSONB:
+ tmp, err := parseJSON(value, true)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse JSON Array: %w", err)
+ }
+ return tmp, nil
case qvalue.QValueKindInt16:
intVal := value.(int16)
return qvalue.QValueInt16{Val: intVal}, nil
@@ -483,3 +528,23 @@ func customTypeToQKind(typeName string) qvalue.QValueKind {
return qvalue.QValueKindString
}
}
+
+// Postgres does not like timestamps of the form 2006-01-02 15:04:05 +0000 UTC
+// in tstzrange.
+// convertTimeRangeBound removes the +0000 UTC part
+func convertTimeRangeBound(timeBound interface{}) (string, error) {
+ layout := "2006-01-02 15:04:05 -0700 MST"
+ postgresFormat := "2006-01-02 15:04:05"
+ var convertedTime string
+ if timeBound != nil {
+ lowerParsed, err := time.Parse(layout, fmt.Sprint(timeBound))
+ if err != nil {
+ return "", fmt.Errorf("unexpected lower bound value in tstzrange. Error: %v", err)
+ }
+ convertedTime = lowerParsed.Format(postgresFormat)
+ } else {
+ convertedTime = ""
+ }
+
+ return convertedTime, nil
+}
diff --git a/flow/connectors/postgres/sink_q.go b/flow/connectors/postgres/sink_q.go
index 89dab6a94f..21a39627be 100644
--- a/flow/connectors/postgres/sink_q.go
+++ b/flow/connectors/postgres/sink_q.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log/slog"
+ "math/rand/v2"
"github.com/jackc/pgx/v5"
@@ -35,20 +36,15 @@ func (stream RecordStreamSink) ExecuteQueryWithTx(
}
}
- randomUint, err := shared.RandomUInt64()
- if err != nil {
- qe.logger.Error("[pg_query_executor] failed to generate random uint", slog.Any("error", err))
- err = fmt.Errorf("[pg_query_executor] failed to generate random uint: %w", err)
- stream.Close(err)
- return 0, err
- }
+ //nolint:gosec // number has no cryptographic significance
+ randomUint := rand.Uint64()
cursorName := fmt.Sprintf("peerdb_cursor_%d", randomUint)
fetchSize := shared.FetchAndChannelSize
cursorQuery := fmt.Sprintf("DECLARE %s CURSOR FOR %s", cursorName, query)
qe.logger.Info(fmt.Sprintf("[pg_query_executor] executing cursor declaration for %v with args %v", cursorQuery, args))
- _, err = tx.Exec(ctx, cursorQuery, args...)
- if err != nil {
+
+ if _, err := tx.Exec(ctx, cursorQuery, args...); err != nil {
qe.logger.Info("[pg_query_executor] failed to declare cursor",
slog.String("cursorQuery", cursorQuery), slog.Any("error", err))
err = fmt.Errorf("[pg_query_executor] failed to declare cursor: %w", err)
diff --git a/flow/connectors/pubsub/pubsub.go b/flow/connectors/pubsub/pubsub.go
index 49aed379c4..537cda7241 100644
--- a/flow/connectors/pubsub/pubsub.go
+++ b/flow/connectors/pubsub/pubsub.go
@@ -67,7 +67,9 @@ func (c *PubSubConnector) CreateRawTable(ctx context.Context, req *protos.Create
return &protos.CreateRawTableOutput{TableIdentifier: "n/a"}, nil
}
-func (c *PubSubConnector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error {
+func (c *PubSubConnector) ReplayTableSchemaDeltas(_ context.Context, _ map[string]string,
+ flowJobName string, schemaDeltas []*protos.TableSchemaDelta,
+) error {
return nil
}
diff --git a/flow/connectors/s3/qrep.go b/flow/connectors/s3/qrep.go
index 14c7b31ef2..968c956aab 100644
--- a/flow/connectors/s3/qrep.go
+++ b/flow/connectors/s3/qrep.go
@@ -20,12 +20,12 @@ func (c *S3Connector) SyncQRepRecords(
schema := stream.Schema()
dstTableName := config.DestinationTableIdentifier
- avroSchema, err := getAvroSchema(dstTableName, schema)
+ avroSchema, err := getAvroSchema(ctx, config.Env, dstTableName, schema)
if err != nil {
return 0, err
}
- numRecords, err := c.writeToAvroFile(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName)
+ numRecords, err := c.writeToAvroFile(ctx, config.Env, stream, avroSchema, partition.PartitionId, config.FlowJobName)
if err != nil {
return 0, err
}
@@ -34,10 +34,12 @@ func (c *S3Connector) SyncQRepRecords(
}
func getAvroSchema(
+ ctx context.Context,
+ env map[string]string,
dstTableName string,
schema qvalue.QRecordSchema,
) (*model.QRecordAvroSchemaDefinition, error) {
- avroSchema, err := model.GetAvroSchemaDefinition(dstTableName, schema, protos.DBType_S3)
+ avroSchema, err := model.GetAvroSchemaDefinition(ctx, env, dstTableName, schema, protos.DBType_S3)
if err != nil {
return nil, fmt.Errorf("failed to define Avro schema: %w", err)
}
@@ -47,6 +49,7 @@ func getAvroSchema(
func (c *S3Connector) writeToAvroFile(
ctx context.Context,
+ env map[string]string,
stream *model.QRecordStream,
avroSchema *model.QRecordAvroSchemaDefinition,
partitionID string,
@@ -60,7 +63,7 @@ func (c *S3Connector) writeToAvroFile(
s3AvroFileKey := fmt.Sprintf("%s/%s/%s.avro", s3o.Prefix, jobName, partitionID)
writer := avro.NewPeerDBOCFWriter(stream, avroSchema, avro.CompressNone, protos.DBType_SNOWFLAKE)
- avroFile, err := writer.WriteRecordsToS3(ctx, s3o.Bucket, s3AvroFileKey, c.credentialsProvider)
+ avroFile, err := writer.WriteRecordsToS3(ctx, env, s3o.Bucket, s3AvroFileKey, c.credentialsProvider)
if err != nil {
return 0, fmt.Errorf("failed to write records to S3: %w", err)
}
diff --git a/flow/connectors/s3/s3.go b/flow/connectors/s3/s3.go
index eac37cd7c8..7d16a20af0 100644
--- a/flow/connectors/s3/s3.go
+++ b/flow/connectors/s3/s3.go
@@ -118,7 +118,9 @@ func (c *S3Connector) SyncRecords(ctx context.Context, req *model.SyncRecordsReq
}, nil
}
-func (c *S3Connector) ReplayTableSchemaDeltas(_ context.Context, flowJobName string, schemaDeltas []*protos.TableSchemaDelta) error {
+func (c *S3Connector) ReplayTableSchemaDeltas(_ context.Context, _ map[string]string,
+ flowJobName string, schemaDeltas []*protos.TableSchemaDelta,
+) error {
c.logger.Info("ReplayTableSchemaDeltas for S3 is a no-op")
return nil
}
diff --git a/flow/connectors/snowflake/avro_file_writer_test.go b/flow/connectors/snowflake/avro_file_writer_test.go
index ac6f253517..4a76fccd01 100644
--- a/flow/connectors/snowflake/avro_file_writer_test.go
+++ b/flow/connectors/snowflake/avro_file_writer_test.go
@@ -144,14 +144,14 @@ func TestWriteRecordsToAvroFileHappyPath(t *testing.T) {
// Define sample data
records, schema := generateRecords(t, true, 10, false)
- avroSchema, err := model.GetAvroSchemaDefinition("not_applicable", schema, protos.DBType_SNOWFLAKE)
+ avroSchema, err := model.GetAvroSchemaDefinition(context.Background(), nil, "not_applicable", schema, protos.DBType_SNOWFLAKE)
require.NoError(t, err)
t.Logf("[test] avroSchema: %v", avroSchema)
// Call function
writer := avro.NewPeerDBOCFWriter(records, avroSchema, avro.CompressNone, protos.DBType_SNOWFLAKE)
- _, err = writer.WriteRecordsToAvroFile(context.Background(), tmpfile.Name())
+ _, err = writer.WriteRecordsToAvroFile(context.Background(), nil, tmpfile.Name())
require.NoError(t, err, "expected WriteRecordsToAvroFile to complete without errors")
// Check file is not empty
@@ -171,14 +171,14 @@ func TestWriteRecordsToZstdAvroFileHappyPath(t *testing.T) {
// Define sample data
records, schema := generateRecords(t, true, 10, false)
- avroSchema, err := model.GetAvroSchemaDefinition("not_applicable", schema, protos.DBType_SNOWFLAKE)
+ avroSchema, err := model.GetAvroSchemaDefinition(context.Background(), nil, "not_applicable", schema, protos.DBType_SNOWFLAKE)
require.NoError(t, err)
t.Logf("[test] avroSchema: %v", avroSchema)
// Call function
writer := avro.NewPeerDBOCFWriter(records, avroSchema, avro.CompressZstd, protos.DBType_SNOWFLAKE)
- _, err = writer.WriteRecordsToAvroFile(context.Background(), tmpfile.Name())
+ _, err = writer.WriteRecordsToAvroFile(context.Background(), nil, tmpfile.Name())
require.NoError(t, err, "expected WriteRecordsToAvroFile to complete without errors")
// Check file is not empty
@@ -198,14 +198,14 @@ func TestWriteRecordsToDeflateAvroFileHappyPath(t *testing.T) {
// Define sample data
records, schema := generateRecords(t, true, 10, false)
- avroSchema, err := model.GetAvroSchemaDefinition("not_applicable", schema, protos.DBType_SNOWFLAKE)
+ avroSchema, err := model.GetAvroSchemaDefinition(context.Background(), nil, "not_applicable", schema, protos.DBType_SNOWFLAKE)
require.NoError(t, err)
t.Logf("[test] avroSchema: %v", avroSchema)
// Call function
writer := avro.NewPeerDBOCFWriter(records, avroSchema, avro.CompressDeflate, protos.DBType_SNOWFLAKE)
- _, err = writer.WriteRecordsToAvroFile(context.Background(), tmpfile.Name())
+ _, err = writer.WriteRecordsToAvroFile(context.Background(), nil, tmpfile.Name())
require.NoError(t, err, "expected WriteRecordsToAvroFile to complete without errors")
// Check file is not empty
@@ -224,14 +224,14 @@ func TestWriteRecordsToAvroFileNonNull(t *testing.T) {
records, schema := generateRecords(t, false, 10, false)
- avroSchema, err := model.GetAvroSchemaDefinition("not_applicable", schema, protos.DBType_SNOWFLAKE)
+ avroSchema, err := model.GetAvroSchemaDefinition(context.Background(), nil, "not_applicable", schema, protos.DBType_SNOWFLAKE)
require.NoError(t, err)
t.Logf("[test] avroSchema: %v", avroSchema)
// Call function
writer := avro.NewPeerDBOCFWriter(records, avroSchema, avro.CompressNone, protos.DBType_SNOWFLAKE)
- _, err = writer.WriteRecordsToAvroFile(context.Background(), tmpfile.Name())
+ _, err = writer.WriteRecordsToAvroFile(context.Background(), nil, tmpfile.Name())
require.NoError(t, err, "expected WriteRecordsToAvroFile to complete without errors")
// Check file is not empty
@@ -251,14 +251,14 @@ func TestWriteRecordsToAvroFileAllNulls(t *testing.T) {
// Define sample data
records, schema := generateRecords(t, true, 10, true)
- avroSchema, err := model.GetAvroSchemaDefinition("not_applicable", schema, protos.DBType_SNOWFLAKE)
+ avroSchema, err := model.GetAvroSchemaDefinition(context.Background(), nil, "not_applicable", schema, protos.DBType_SNOWFLAKE)
require.NoError(t, err)
t.Logf("[test] avroSchema: %v", avroSchema)
// Call function
writer := avro.NewPeerDBOCFWriter(records, avroSchema, avro.CompressNone, protos.DBType_SNOWFLAKE)
- _, err = writer.WriteRecordsToAvroFile(context.Background(), tmpfile.Name())
+ _, err = writer.WriteRecordsToAvroFile(context.Background(), nil, tmpfile.Name())
require.NoError(t, err, "expected WriteRecordsToAvroFile to complete without errors")
// Check file is not empty
diff --git a/flow/connectors/snowflake/merge_stmt_generator.go b/flow/connectors/snowflake/merge_stmt_generator.go
index 3f0cfbc63a..d87d3004f7 100644
--- a/flow/connectors/snowflake/merge_stmt_generator.go
+++ b/flow/connectors/snowflake/merge_stmt_generator.go
@@ -1,6 +1,7 @@
package connsnowflake
import (
+ "context"
"fmt"
"strings"
@@ -24,7 +25,7 @@ type mergeStmtGenerator struct {
mergeBatchId int64
}
-func (m *mergeStmtGenerator) generateMergeStmt(dstTable string) (string, error) {
+func (m *mergeStmtGenerator) generateMergeStmt(ctx context.Context, env map[string]string, dstTable string) (string, error) {
parsedDstTable, _ := utils.ParseSchemaTable(dstTable)
normalizedTableSchema := m.tableSchemaMapping[dstTable]
unchangedToastColumns := m.unchangedToastColumnsMap[dstTable]
@@ -34,7 +35,7 @@ func (m *mergeStmtGenerator) generateMergeStmt(dstTable string) (string, error)
for _, column := range columns {
genericColumnType := column.Type
qvKind := qvalue.QValueKind(genericColumnType)
- sfType, err := qvKind.ToDWHColumnType(protos.DBType_SNOWFLAKE)
+ sfType, err := qvKind.ToDWHColumnType(ctx, env, protos.DBType_SNOWFLAKE, column)
if err != nil {
return "", fmt.Errorf("failed to convert column type %s to snowflake type: %w", genericColumnType, err)
}
@@ -52,7 +53,7 @@ func (m *mergeStmtGenerator) generateMergeStmt(dstTable string) (string, error)
flattenedCastsSQLArray = append(flattenedCastsSQLArray,
fmt.Sprintf("TO_GEOMETRY(CAST(%s:\"%s\" AS STRING),true) AS %s",
toVariantColumnName, column.Name, targetColumnName))
- case qvalue.QValueKindJSON, qvalue.QValueKindHStore, qvalue.QValueKindInterval:
+ case qvalue.QValueKindJSON, qvalue.QValueKindJSONB, qvalue.QValueKindHStore, qvalue.QValueKindInterval:
flattenedCastsSQLArray = append(flattenedCastsSQLArray,
fmt.Sprintf("PARSE_JSON(CAST(%s:\"%s\" AS STRING)) AS %s",
toVariantColumnName, column.Name, targetColumnName))
diff --git a/flow/connectors/snowflake/qrep_avro_consolidate.go b/flow/connectors/snowflake/qrep_avro_consolidate.go
index 547aef27ef..a4a8d1a285 100644
--- a/flow/connectors/snowflake/qrep_avro_consolidate.go
+++ b/flow/connectors/snowflake/qrep_avro_consolidate.go
@@ -4,13 +4,13 @@ import (
"context"
"fmt"
"log/slog"
+ "math/rand/v2"
"strings"
"time"
"github.com/PeerDB-io/peer-flow/connectors/utils"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/peerdbenv"
- "github.com/PeerDB-io/peer-flow/shared"
)
type SnowflakeAvroConsolidateHandler struct {
@@ -214,10 +214,8 @@ func (s *SnowflakeAvroConsolidateHandler) generateUpsertMergeCommand(
// handleUpsertMode handles the upsert mode
func (s *SnowflakeAvroConsolidateHandler) handleUpsertMode(ctx context.Context) error {
- runID, err := shared.RandomUInt64()
- if err != nil {
- return fmt.Errorf("failed to generate run ID: %w", err)
- }
+ //nolint:gosec // number has no cryptographic significance
+ runID := rand.Uint64()
tempTableName := fmt.Sprintf("%s_temp_%d", s.dstTableName, runID)
@@ -230,8 +228,8 @@ func (s *SnowflakeAvroConsolidateHandler) handleUpsertMode(ctx context.Context)
s.connector.logger.Info("created temp table " + tempTableName)
copyCmd := s.getCopyTransformation(tempTableName)
- _, err = s.connector.database.ExecContext(ctx, copyCmd)
- if err != nil {
+
+ if _, err := s.connector.database.ExecContext(ctx, copyCmd); err != nil {
return fmt.Errorf("failed to run COPY INTO command: %w", err)
}
s.connector.logger.Info("copied file from stage " + s.stage + " to temp table " + tempTableName)
diff --git a/flow/connectors/snowflake/qrep_avro_sync.go b/flow/connectors/snowflake/qrep_avro_sync.go
index 2e37705c14..728d393e62 100644
--- a/flow/connectors/snowflake/qrep_avro_sync.go
+++ b/flow/connectors/snowflake/qrep_avro_sync.go
@@ -20,8 +20,8 @@ import (
)
type SnowflakeAvroSyncHandler struct {
- config *protos.QRepConfig
- connector *SnowflakeConnector
+ *SnowflakeConnector
+ config *protos.QRepConfig
}
func NewSnowflakeAvroSyncHandler(
@@ -29,13 +29,14 @@ func NewSnowflakeAvroSyncHandler(
connector *SnowflakeConnector,
) *SnowflakeAvroSyncHandler {
return &SnowflakeAvroSyncHandler{
- config: config,
- connector: connector,
+ SnowflakeConnector: connector,
+ config: config,
}
}
func (s *SnowflakeAvroSyncHandler) SyncRecords(
ctx context.Context,
+ env map[string]string,
dstTableSchema []*sql.ColumnType,
stream *model.QRecordStream,
flowJobName string,
@@ -45,40 +46,39 @@ func (s *SnowflakeAvroSyncHandler) SyncRecords(
schema := stream.Schema()
- s.connector.logger.Info("sync function called and schema acquired", tableLog)
+ s.logger.Info("sync function called and schema acquired", tableLog)
- avroSchema, err := s.getAvroSchema(dstTableName, schema)
+ avroSchema, err := s.getAvroSchema(ctx, env, dstTableName, schema)
if err != nil {
return 0, err
}
partitionID := shared.RandomString(16)
- avroFile, err := s.writeToAvroFile(ctx, stream, avroSchema, partitionID, flowJobName)
+ avroFile, err := s.writeToAvroFile(ctx, env, stream, avroSchema, partitionID, flowJobName)
if err != nil {
return 0, err
}
defer avroFile.Cleanup()
- s.connector.logger.Info(fmt.Sprintf("written %d records to Avro file", avroFile.NumRecords), tableLog)
+ s.logger.Info(fmt.Sprintf("written %d records to Avro file", avroFile.NumRecords), tableLog)
- stage := s.connector.getStageNameForJob(s.config.FlowJobName)
- err = s.connector.createStage(ctx, stage, s.config)
- if err != nil {
+ stage := s.getStageNameForJob(s.config.FlowJobName)
+ if err := s.createStage(ctx, stage, s.config); err != nil {
return 0, err
}
- s.connector.logger.Info("Created stage " + stage)
+ s.logger.Info("Created stage " + stage)
err = s.putFileToStage(ctx, avroFile, stage)
if err != nil {
return 0, err
}
- s.connector.logger.Info("pushed avro file to stage", tableLog)
+ s.logger.Info("pushed avro file to stage", tableLog)
- writeHandler := NewSnowflakeAvroConsolidateHandler(s.connector, s.config, s.config.DestinationTableIdentifier, stage)
+ writeHandler := NewSnowflakeAvroConsolidateHandler(s.SnowflakeConnector, s.config, s.config.DestinationTableIdentifier, stage)
err = writeHandler.CopyStageToDestination(ctx)
if err != nil {
return 0, err
}
- s.connector.logger.Info(fmt.Sprintf("copying records into %s from stage %s",
+ s.logger.Info(fmt.Sprintf("copying records into %s from stage %s",
s.config.DestinationTableIdentifier, stage))
return avroFile.NumRecords, nil
@@ -96,34 +96,32 @@ func (s *SnowflakeAvroSyncHandler) SyncQRepRecords(
dstTableName := config.DestinationTableIdentifier
schema := stream.Schema()
- s.connector.logger.Info("sync function called and schema acquired", partitionLog)
+ s.logger.Info("sync function called and schema acquired", partitionLog)
- err := s.addMissingColumns(ctx, schema, dstTableSchema, dstTableName, partition)
+ err := s.addMissingColumns(ctx, config.Env, schema, dstTableSchema, dstTableName, partition)
if err != nil {
return 0, err
}
- avroSchema, err := s.getAvroSchema(dstTableName, schema)
+ avroSchema, err := s.getAvroSchema(ctx, config.Env, dstTableName, schema)
if err != nil {
return 0, err
}
- avroFile, err := s.writeToAvroFile(ctx, stream, avroSchema, partition.PartitionId, config.FlowJobName)
+ avroFile, err := s.writeToAvroFile(ctx, config.Env, stream, avroSchema, partition.PartitionId, config.FlowJobName)
if err != nil {
return 0, err
}
defer avroFile.Cleanup()
- stage := s.connector.getStageNameForJob(config.FlowJobName)
+ stage := s.getStageNameForJob(config.FlowJobName)
- err = s.putFileToStage(ctx, avroFile, stage)
- if err != nil {
+ if err := s.putFileToStage(ctx, avroFile, stage); err != nil {
return 0, err
}
- s.connector.logger.Info("Put file to stage in Avro sync for snowflake", partitionLog)
+ s.logger.Info("Put file to stage in Avro sync for snowflake", partitionLog)
- err = s.connector.FinishQRepPartition(ctx, partition, config.FlowJobName, startTime)
- if err != nil {
+ if err := s.FinishQRepPartition(ctx, partition, config.FlowJobName, startTime); err != nil {
return 0, err
}
@@ -132,6 +130,7 @@ func (s *SnowflakeAvroSyncHandler) SyncQRepRecords(
func (s *SnowflakeAvroSyncHandler) addMissingColumns(
ctx context.Context,
+ env map[string]string,
schema qvalue.QRecordSchema,
dstTableSchema []*sql.ColumnType,
dstTableName string,
@@ -140,7 +139,7 @@ func (s *SnowflakeAvroSyncHandler) addMissingColumns(
partitionLog := slog.String(string(shared.PartitionIDKey), partition.PartitionId)
// check if avro schema has additional columns compared to destination table
// if so, we need to add those columns to the destination table
- colsToTypes := map[string]qvalue.QValueKind{}
+ var newColumns []qvalue.QField
for _, col := range schema.Fields {
hasColumn := false
// check ignoring case
@@ -152,28 +151,27 @@ func (s *SnowflakeAvroSyncHandler) addMissingColumns(
}
if !hasColumn {
- s.connector.logger.Info(fmt.Sprintf("adding column %s to destination table %s",
+ s.logger.Info(fmt.Sprintf("adding column %s to destination table %s",
col.Name, dstTableName), partitionLog)
- colsToTypes[col.Name] = col.Type
+ newColumns = append(newColumns, col)
}
}
- if len(colsToTypes) > 0 {
- tx, err := s.connector.database.Begin()
+ if len(newColumns) > 0 {
+ tx, err := s.database.Begin()
if err != nil {
return fmt.Errorf("failed to begin transaction: %w", err)
}
- for colName, colType := range colsToTypes {
- sfColType, err := colType.ToDWHColumnType(protos.DBType_SNOWFLAKE)
+ for _, column := range newColumns {
+ sfColType, err := column.ToDWHColumnType(ctx, env, protos.DBType_SNOWFLAKE)
if err != nil {
return fmt.Errorf("failed to convert QValueKind to Snowflake column type: %w", err)
}
- upperCasedColName := strings.ToUpper(colName)
- alterTableCmd := fmt.Sprintf("ALTER TABLE %s ", dstTableName)
- alterTableCmd += fmt.Sprintf("ADD COLUMN IF NOT EXISTS \"%s\" %s;", upperCasedColName, sfColType)
+ upperCasedColName := strings.ToUpper(column.Name)
+ alterTableCmd := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS \"%s\" %s;", dstTableName, upperCasedColName, sfColType)
- s.connector.logger.Info(fmt.Sprintf("altering destination table %s with command `%s`",
+ s.logger.Info(fmt.Sprintf("altering destination table %s with command `%s`",
dstTableName, alterTableCmd), partitionLog)
if _, err := tx.ExecContext(ctx, alterTableCmd); err != nil {
@@ -185,30 +183,33 @@ func (s *SnowflakeAvroSyncHandler) addMissingColumns(
return fmt.Errorf("failed to commit transaction: %w", err)
}
- s.connector.logger.Info("successfully added missing columns to destination table "+
+ s.logger.Info("successfully added missing columns to destination table "+
dstTableName, partitionLog)
} else {
- s.connector.logger.Info("no missing columns found in destination table "+dstTableName, partitionLog)
+ s.logger.Info("no missing columns found in destination table "+dstTableName, partitionLog)
}
return nil
}
func (s *SnowflakeAvroSyncHandler) getAvroSchema(
+ ctx context.Context,
+ env map[string]string,
dstTableName string,
schema qvalue.QRecordSchema,
) (*model.QRecordAvroSchemaDefinition, error) {
- avroSchema, err := model.GetAvroSchemaDefinition(dstTableName, schema, protos.DBType_SNOWFLAKE)
+ avroSchema, err := model.GetAvroSchemaDefinition(ctx, env, dstTableName, schema, protos.DBType_SNOWFLAKE)
if err != nil {
return nil, fmt.Errorf("failed to define Avro schema: %w", err)
}
- s.connector.logger.Info(fmt.Sprintf("Avro schema: %v\n", avroSchema))
+ s.logger.Info(fmt.Sprintf("Avro schema: %v\n", avroSchema))
return avroSchema, nil
}
func (s *SnowflakeAvroSyncHandler) writeToAvroFile(
ctx context.Context,
+ env map[string]string,
stream *model.QRecordStream,
avroSchema *model.QRecordAvroSchemaDefinition,
partitionID string,
@@ -223,8 +224,8 @@ func (s *SnowflakeAvroSyncHandler) writeToAvroFile(
}
localFilePath := fmt.Sprintf("%s/%s.avro.zst", tmpDir, partitionID)
- s.connector.logger.Info("writing records to local file " + localFilePath)
- avroFile, err := ocfWriter.WriteRecordsToAvroFile(ctx, localFilePath)
+ s.logger.Info("writing records to local file " + localFilePath)
+ avroFile, err := ocfWriter.WriteRecordsToAvroFile(ctx, env, localFilePath)
if err != nil {
return nil, fmt.Errorf("failed to write records to Avro file: %w", err)
}
@@ -238,14 +239,14 @@ func (s *SnowflakeAvroSyncHandler) writeToAvroFile(
}
s3AvroFileKey := fmt.Sprintf("%s/%s/%s.avro.zst", s3o.Prefix, s.config.FlowJobName, partitionID)
- s.connector.logger.Info("OCF: Writing records to S3",
+ s.logger.Info("OCF: Writing records to S3",
slog.String(string(shared.PartitionIDKey), partitionID))
provider, err := utils.GetAWSCredentialsProvider(ctx, "snowflake", utils.PeerAWSCredentials{})
if err != nil {
return nil, err
}
- avroFile, err := ocfWriter.WriteRecordsToS3(ctx, s3o.Bucket, s3AvroFileKey, provider)
+ avroFile, err := ocfWriter.WriteRecordsToS3(ctx, env, s3o.Bucket, s3AvroFileKey, provider)
if err != nil {
return nil, fmt.Errorf("failed to write records to S3: %w", err)
}
@@ -258,16 +259,16 @@ func (s *SnowflakeAvroSyncHandler) writeToAvroFile(
func (s *SnowflakeAvroSyncHandler) putFileToStage(ctx context.Context, avroFile *avro.AvroFile, stage string) error {
if avroFile.StorageLocation != avro.AvroLocalStorage {
- s.connector.logger.Info("no file to put to stage")
+ s.logger.Info("no file to put to stage")
return nil
}
putCmd := fmt.Sprintf("PUT file://%s @%s", avroFile.FilePath, stage)
- if _, err := s.connector.database.ExecContext(ctx, putCmd); err != nil {
+ if _, err := s.database.ExecContext(ctx, putCmd); err != nil {
return fmt.Errorf("failed to put file to stage: %w", err)
}
- s.connector.logger.Info(fmt.Sprintf("put file %s to stage %s", avroFile.FilePath, stage))
+ s.logger.Info(fmt.Sprintf("put file %s to stage %s", avroFile.FilePath, stage))
return nil
}
diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go
index 7a400d78a7..518b01ff2b 100644
--- a/flow/connectors/snowflake/snowflake.go
+++ b/flow/connectors/snowflake/snowflake.go
@@ -19,7 +19,6 @@ import (
metadataStore "github.com/PeerDB-io/peer-flow/connectors/external_metadata"
"github.com/PeerDB-io/peer-flow/connectors/utils"
- numeric "github.com/PeerDB-io/peer-flow/datatypes"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/model/qvalue"
@@ -338,7 +337,7 @@ func (c *SnowflakeConnector) SetupNormalizedTable(
return true, nil
}
- normalizedTableCreateSQL := generateCreateTableSQLForNormalizedTable(config, normalizedSchemaTable, tableSchema)
+ normalizedTableCreateSQL := generateCreateTableSQLForNormalizedTable(ctx, config, normalizedSchemaTable, tableSchema)
if _, err := c.execWithLogging(ctx, normalizedTableCreateSQL); err != nil {
return false, fmt.Errorf("[sf] error while creating normalized table: %w", err)
}
@@ -349,6 +348,7 @@ func (c *SnowflakeConnector) SetupNormalizedTable(
// This could involve adding or dropping multiple columns.
func (c *SnowflakeConnector) ReplayTableSchemaDeltas(
ctx context.Context,
+ env map[string]string,
flowJobName string,
schemaDeltas []*protos.TableSchemaDelta,
) error {
@@ -374,17 +374,12 @@ func (c *SnowflakeConnector) ReplayTableSchemaDeltas(
}
for _, addedColumn := range schemaDelta.AddedColumns {
- sfColtype, err := qvalue.QValueKind(addedColumn.Type).ToDWHColumnType(protos.DBType_SNOWFLAKE)
+ sfColtype, err := qvalue.QValueKind(addedColumn.Type).ToDWHColumnType(ctx, env, protos.DBType_SNOWFLAKE, addedColumn)
if err != nil {
return fmt.Errorf("failed to convert column type %s to snowflake type: %w",
addedColumn.Type, err)
}
- if addedColumn.Type == string(qvalue.QValueKindNumeric) {
- precision, scale := numeric.GetNumericTypeForWarehouse(addedColumn.TypeModifier, numeric.SnowflakeNumericCompatibility{})
- sfColtype = fmt.Sprintf("NUMERIC(%d,%d)", precision, scale)
- }
-
_, err = tableSchemaModifyTx.ExecContext(ctx,
fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS \"%s\" %s",
schemaDelta.DstTableName, strings.ToUpper(addedColumn.Name), sfColtype))
@@ -423,8 +418,7 @@ func (c *SnowflakeConnector) SyncRecords(ctx context.Context, req *model.SyncRec
return nil, err
}
- err = c.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, res.LastSyncedCheckpointID)
- if err != nil {
+ if err := c.FinishBatch(ctx, req.FlowJobName, req.SyncBatchID, res.LastSyncedCheckpointID); err != nil {
return nil, err
}
@@ -456,12 +450,12 @@ func (c *SnowflakeConnector) syncRecordsViaAvro(
return nil, err
}
- numRecords, err := avroSyncer.SyncRecords(ctx, destinationTableSchema, stream, req.FlowJobName)
+ numRecords, err := avroSyncer.SyncRecords(ctx, req.Env, destinationTableSchema, stream, req.FlowJobName)
if err != nil {
return nil, err
}
- err = c.ReplayTableSchemaDeltas(ctx, req.FlowJobName, req.Records.SchemaDeltas)
+ err = c.ReplayTableSchemaDeltas(ctx, req.Env, req.FlowJobName, req.Records.SchemaDeltas)
if err != nil {
return nil, fmt.Errorf("failed to sync schema changes: %w", err)
}
@@ -558,7 +552,7 @@ func (c *SnowflakeConnector) mergeTablesForBatch(
}
g.Go(func() error {
- mergeStatement, err := mergeGen.generateMergeStmt(tableName)
+ mergeStatement, err := mergeGen.generateMergeStmt(gCtx, env, tableName)
if err != nil {
return err
}
@@ -667,6 +661,7 @@ func (c *SnowflakeConnector) checkIfTableExists(
}
func generateCreateTableSQLForNormalizedTable(
+ ctx context.Context,
config *protos.SetupNormalizedTableBatchInput,
dstSchemaTable *utils.SchemaTable,
tableSchema *protos.TableSchema,
@@ -675,18 +670,13 @@ func generateCreateTableSQLForNormalizedTable(
for _, column := range tableSchema.Columns {
genericColumnType := column.Type
normalizedColName := SnowflakeIdentifierNormalize(column.Name)
- sfColType, err := qvalue.QValueKind(genericColumnType).ToDWHColumnType(protos.DBType_SNOWFLAKE)
+ sfColType, err := qvalue.QValueKind(genericColumnType).ToDWHColumnType(ctx, config.Env, protos.DBType_SNOWFLAKE, column)
if err != nil {
slog.Warn(fmt.Sprintf("failed to convert column type %s to snowflake type", genericColumnType),
slog.Any("error", err))
continue
}
- if genericColumnType == "numeric" {
- precision, scale := numeric.GetNumericTypeForWarehouse(column.TypeModifier, numeric.SnowflakeNumericCompatibility{})
- sfColType = fmt.Sprintf("NUMERIC(%d,%d)", precision, scale)
- }
-
var notNull string
if tableSchema.NullableEnabled && !column.Nullable {
notNull = " NOT NULL"
diff --git a/flow/connectors/utils/avro/avro_writer.go b/flow/connectors/utils/avro/avro_writer.go
index 6f193be88b..75bc9f4358 100644
--- a/flow/connectors/utils/avro/avro_writer.go
+++ b/flow/connectors/utils/avro/avro_writer.go
@@ -23,6 +23,7 @@ import (
"github.com/PeerDB-io/peer-flow/connectors/utils"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
)
@@ -126,16 +127,21 @@ func (p *peerDBOCFWriter) createOCFWriter(w io.Writer) (*goavro.OCFWriter, error
return ocfWriter, nil
}
-func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter *goavro.OCFWriter) (int64, error) {
+func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, env map[string]string, ocfWriter *goavro.OCFWriter) (int64, error) {
logger := shared.LoggerFromCtx(ctx)
schema := p.stream.Schema()
- avroConverter := model.NewQRecordAvroConverter(
+ avroConverter, err := model.NewQRecordAvroConverter(
+ ctx,
+ env,
p.avroSchema,
p.targetDWH,
schema.GetColumnNames(),
logger,
)
+ if err != nil {
+ return 0, err
+ }
numRows := atomic.Int64{}
@@ -146,7 +152,7 @@ func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter
for qrecord := range p.stream.Records {
if err := ctx.Err(); err != nil {
- return numRows.Load(), ctx.Err()
+ return numRows.Load(), err
} else {
avroMap, err := avroConverter.Convert(qrecord)
if err != nil {
@@ -171,7 +177,7 @@ func (p *peerDBOCFWriter) writeRecordsToOCFWriter(ctx context.Context, ocfWriter
return numRows.Load(), nil
}
-func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer) (int, error) {
+func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, env map[string]string, w io.Writer) (int, error) {
ocfWriter, err := p.createOCFWriter(w)
if err != nil {
return 0, fmt.Errorf("failed to create OCF writer: %w", err)
@@ -179,7 +185,7 @@ func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer) (int, error
// we have to keep a reference to the underlying writer as goavro doesn't provide any access to it
defer p.writer.Close()
- numRows, err := p.writeRecordsToOCFWriter(ctx, ocfWriter)
+ numRows, err := p.writeRecordsToOCFWriter(ctx, env, ocfWriter)
if err != nil {
return 0, fmt.Errorf("failed to write records to OCF writer: %w", err)
}
@@ -187,7 +193,11 @@ func (p *peerDBOCFWriter) WriteOCF(ctx context.Context, w io.Writer) (int, error
}
func (p *peerDBOCFWriter) WriteRecordsToS3(
- ctx context.Context, bucketName, key string, s3Creds utils.AWSCredentialsProvider,
+ ctx context.Context,
+ env map[string]string,
+ bucketName string,
+ key string,
+ s3Creds utils.AWSCredentialsProvider,
) (*AvroFile, error) {
logger := shared.LoggerFromCtx(ctx)
s3svc, err := utils.CreateS3Client(ctx, s3Creds)
@@ -212,15 +222,26 @@ func (p *peerDBOCFWriter) WriteRecordsToS3(
}
w.Close()
}()
- numRows, writeOcfError = p.WriteOCF(ctx, w)
+ numRows, writeOcfError = p.WriteOCF(ctx, env, w)
}()
- _, err = manager.NewUploader(s3svc).Upload(ctx, &s3.PutObjectInput{
+ partSize, err := peerdbenv.PeerDBS3PartSize(ctx, env)
+ if err != nil {
+ return nil, fmt.Errorf("could not get s3 part size config: %w", err)
+ }
+
+ // Create the uploader using the AWS SDK v2 manager
+ uploader := manager.NewUploader(s3svc, func(u *manager.Uploader) {
+ if partSize > 0 {
+ u.PartSize = partSize
+ }
+ })
+
+ if _, err := uploader.Upload(ctx, &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(key),
Body: r,
- })
- if err != nil {
+ }); err != nil {
s3Path := "s3://" + bucketName + "/" + key
logger.Error("failed to upload file", slog.Any("error", err), slog.String("s3_path", s3Path))
return nil, fmt.Errorf("failed to upload file: %w", err)
@@ -238,7 +259,7 @@ func (p *peerDBOCFWriter) WriteRecordsToS3(
}, nil
}
-func (p *peerDBOCFWriter) WriteRecordsToAvroFile(ctx context.Context, filePath string) (*AvroFile, error) {
+func (p *peerDBOCFWriter) WriteRecordsToAvroFile(ctx context.Context, env map[string]string, filePath string) (*AvroFile, error) {
file, err := os.Create(filePath)
if err != nil {
return nil, fmt.Errorf("failed to create temporary Avro file: %w", err)
@@ -259,7 +280,7 @@ func (p *peerDBOCFWriter) WriteRecordsToAvroFile(ctx context.Context, filePath s
bufferedWriter := bufio.NewWriterSize(file, buffSizeBytes)
defer bufferedWriter.Flush()
- numRecords, err := p.WriteOCF(ctx, bufferedWriter)
+ numRecords, err := p.WriteOCF(ctx, env, bufferedWriter)
if err != nil {
return nil, fmt.Errorf("failed to write records to temporary Avro file: %w", err)
}
diff --git a/flow/connectors/utils/azure.go b/flow/connectors/utils/azure.go
deleted file mode 100644
index df612b47d3..0000000000
--- a/flow/connectors/utils/azure.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package utils
-
-import (
- "errors"
- "os"
-)
-
-func GetAzureSubscriptionID() (string, error) {
- // get this from env
- id := os.Getenv("AZURE_SUBSCRIPTION_ID")
- if id == "" {
- return "", errors.New("AZURE_SUBSCRIPTION_ID is not set")
- }
- return id, nil
-}
diff --git a/flow/connectors/utils/cdc_store.go b/flow/connectors/utils/cdc_store.go
index e3aa9e4499..d3e9d27f8d 100644
--- a/flow/connectors/utils/cdc_store.go
+++ b/flow/connectors/utils/cdc_store.go
@@ -115,6 +115,7 @@ func init() {
gob.Register(qvalue.QValueArrayTimestamp{})
gob.Register(qvalue.QValueArrayTimestampTZ{})
gob.Register(qvalue.QValueArrayBoolean{})
+ gob.Register(qvalue.QValueTSTZRange{})
}
func (c *cdcStore[T]) initPebbleDB() error {
diff --git a/flow/connectors/utils/monitoring/monitoring.go b/flow/connectors/utils/monitoring/monitoring.go
index 9c73970049..98a62ec65b 100644
--- a/flow/connectors/utils/monitoring/monitoring.go
+++ b/flow/connectors/utils/monitoring/monitoring.go
@@ -96,8 +96,10 @@ func UpdateEndTimeForCDCBatch(
batchID int64,
) error {
_, err := pool.Exec(ctx,
- "UPDATE peerdb_stats.cdc_batches SET end_time=$1 WHERE flow_name=$2 AND batch_id=$3",
- time.Now(), flowJobName, batchID)
+ `UPDATE peerdb_stats.cdc_batches
+ SET end_time = COALESCE(end_time, NOW())
+ WHERE flow_name = $1 AND batch_id <= $2`,
+ flowJobName, batchID)
if err != nil {
return fmt.Errorf("error while updating batch in cdc_batch: %w", err)
}
diff --git a/flow/datatypes/numeric.go b/flow/datatypes/numeric.go
index 56c1b17839..8b942e4f67 100644
--- a/flow/datatypes/numeric.go
+++ b/flow/datatypes/numeric.go
@@ -90,6 +90,10 @@ func MakeNumericTypmod(precision int32, scale int32) int32 {
// This is to reverse what make_numeric_typmod of Postgres does:
// https://github.com/postgres/postgres/blob/21912e3c0262e2cfe64856e028799d6927862563/src/backend/utils/adt/numeric.c#L897
func ParseNumericTypmod(typmod int32) (int16, int16) {
+ if typmod == -1 {
+ return 0, 0
+ }
+
offsetMod := typmod - VARHDRSZ
precision := int16((offsetMod >> 16) & 0x7FFF)
scale := int16(offsetMod & 0x7FFF)
@@ -102,6 +106,14 @@ func GetNumericTypeForWarehouse(typmod int32, warehouseNumeric WarehouseNumericC
}
precision, scale := ParseNumericTypmod(typmod)
+ return GetNumericTypeForWarehousePrecisionScale(precision, scale, warehouseNumeric)
+}
+
+func GetNumericTypeForWarehousePrecisionScale(precision int16, scale int16, warehouseNumeric WarehouseNumericCompatibility) (int16, int16) {
+ if precision == 0 && scale == 0 {
+ return warehouseNumeric.DefaultPrecisionAndScale()
+ }
+
if !IsValidPrecision(precision, warehouseNumeric) {
precision = warehouseNumeric.MaxPrecision()
}
diff --git a/flow/e2e/bigquery/bigquery_helper.go b/flow/e2e/bigquery/bigquery_helper.go
index 1ee303acf8..ee33f2bfc3 100644
--- a/flow/e2e/bigquery/bigquery_helper.go
+++ b/flow/e2e/bigquery/bigquery_helper.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"math/big"
+ "math/rand/v2"
"os"
"strings"
"testing"
@@ -21,7 +22,6 @@ import (
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/model/qvalue"
- "github.com/PeerDB-io/peer-flow/shared"
)
type BigQueryTestHelper struct {
@@ -37,10 +37,8 @@ type BigQueryTestHelper struct {
func NewBigQueryTestHelper(t *testing.T) (*BigQueryTestHelper, error) {
t.Helper()
// random 64 bit int to namespace stateful schemas.
- runID, err := shared.RandomUInt64()
- if err != nil {
- return nil, fmt.Errorf("failed to generate random uint64: %w", err)
- }
+ //nolint:gosec // number has no cryptographic significance
+ runID := rand.Uint64()
jsonPath := os.Getenv("TEST_BQ_CREDS")
if jsonPath == "" {
diff --git a/flow/e2e/clickhouse/clickhouse.go b/flow/e2e/clickhouse/clickhouse.go
index 79ff2aa7bb..9756761520 100644
--- a/flow/e2e/clickhouse/clickhouse.go
+++ b/flow/e2e/clickhouse/clickhouse.go
@@ -92,7 +92,7 @@ func (s ClickHouseSuite) Teardown() {
}
func (s ClickHouseSuite) GetRows(table string, cols string) (*model.QRecordBatch, error) {
- ch, err := connclickhouse.Connect(context.Background(), s.Peer().GetClickhouseConfig())
+ ch, err := connclickhouse.Connect(context.Background(), nil, s.Peer().GetClickhouseConfig())
if err != nil {
return nil, err
}
@@ -203,7 +203,7 @@ func SetupSuite(t *testing.T) ClickHouseSuite {
s3Helper: s3Helper,
}
- ch, err := connclickhouse.Connect(context.Background(), s.PeerForDatabase("default").GetClickhouseConfig())
+ ch, err := connclickhouse.Connect(context.Background(), nil, s.PeerForDatabase("default").GetClickhouseConfig())
require.NoError(t, err, "failed to connect to clickhouse")
err = ch.Exec(context.Background(), "CREATE DATABASE e2e_test_"+suffix)
require.NoError(t, err, "failed to create clickhouse database")
diff --git a/flow/e2e/clickhouse/peer_flow_ch_test.go b/flow/e2e/clickhouse/peer_flow_ch_test.go
index 8b28573104..a19e69c8c7 100644
--- a/flow/e2e/clickhouse/peer_flow_ch_test.go
+++ b/flow/e2e/clickhouse/peer_flow_ch_test.go
@@ -4,6 +4,7 @@ import (
"context"
"embed"
"fmt"
+ "strconv"
"strings"
"testing"
"time"
@@ -11,7 +12,7 @@ import (
"github.com/shopspring/decimal"
"github.com/stretchr/testify/require"
- "github.com/PeerDB-io/peer-flow/connectors/clickhouse"
+ connclickhouse "github.com/PeerDB-io/peer-flow/connectors/clickhouse"
"github.com/PeerDB-io/peer-flow/e2e"
"github.com/PeerDB-io/peer-flow/e2eshared"
"github.com/PeerDB-io/peer-flow/generated/protos"
@@ -505,7 +506,7 @@ func (s ClickHouseSuite) WeirdTable(tableName string) {
})
e2e.EnvWaitForFinished(s.t, env, 3*time.Minute)
// now test weird names with rename based resync
- ch, err := connclickhouse.Connect(context.Background(), s.Peer().GetClickhouseConfig())
+ ch, err := connclickhouse.Connect(context.Background(), nil, s.Peer().GetClickhouseConfig())
require.NoError(s.t, err)
require.NoError(s.t, ch.Exec(context.Background(), fmt.Sprintf("DROP TABLE `%s`", dstTableName)))
require.NoError(s.t, ch.Close())
@@ -523,7 +524,7 @@ func (s ClickHouseSuite) WeirdTable(tableName string) {
})
e2e.EnvWaitForFinished(s.t, env, 3*time.Minute)
// now test weird names with exchange based resync
- ch, err = connclickhouse.Connect(context.Background(), s.Peer().GetClickhouseConfig())
+ ch, err = connclickhouse.Connect(context.Background(), nil, s.Peer().GetClickhouseConfig())
require.NoError(s.t, err)
require.NoError(s.t, ch.Exec(context.Background(), fmt.Sprintf("TRUNCATE TABLE `%s`", dstTableName)))
require.NoError(s.t, ch.Close())
@@ -557,8 +558,8 @@ func (s ClickHouseSuite) Test_Large_Numeric() {
`, srcFullName))
require.NoError(s.t, err)
- _, err = s.Conn().Exec(context.Background(), fmt.Sprintf(`
- INSERT INTO %s(c1,c2) VALUES(%s,%s);`, srcFullName, strings.Repeat("7", 76), strings.Repeat("9", 78)))
+ _, err = s.Conn().Exec(context.Background(), fmt.Sprintf("INSERT INTO %s(c1,c2) VALUES($1,$2)", srcFullName),
+ strings.Repeat("7", 76), strings.Repeat("9", 78))
require.NoError(s.t, err)
connectionGen := e2e.FlowConnectionGenerationConfig{
@@ -568,14 +569,15 @@ func (s ClickHouseSuite) Test_Large_Numeric() {
}
flowConnConfig := connectionGen.GenerateFlowConnectionConfigs(s.t)
flowConnConfig.DoInitialSnapshot = true
+
tc := e2e.NewTemporalClient(s.t)
env := e2e.ExecutePeerflow(tc, peerflow.CDCFlowWorkflow, flowConnConfig, nil)
e2e.SetupCDCFlowStatusQuery(s.t, env, flowConnConfig)
e2e.EnvWaitForCount(env, s, "waiting for CDC count", dstTableName, "id,c1,c2", 1)
- _, err = s.Conn().Exec(context.Background(), fmt.Sprintf(`
- INSERT INTO %s(c1,c2) VALUES(%s,%s);`, srcFullName, strings.Repeat("7", 76), strings.Repeat("9", 78)))
+ _, err = s.Conn().Exec(context.Background(), fmt.Sprintf("INSERT INTO %s(c1,c2) VALUES($1,$2)", srcFullName),
+ strings.Repeat("7", 76), strings.Repeat("9", 78))
require.NoError(s.t, err)
e2e.EnvWaitForCount(env, s, "waiting for CDC count", dstTableName, "id,c1,c2", 2)
@@ -598,3 +600,67 @@ func (s ClickHouseSuite) Test_Large_Numeric() {
env.Cancel()
e2e.RequireEnvCanceled(s.t, env)
}
+
+// Unbounded NUMERICs (no precision, scale specified) are mapped to String on CH if FF enabled, Decimal if not
+func (s ClickHouseSuite) testNumericFF(ffValue bool) {
+ nines := strings.Repeat("9", 38)
+ dstTableName := fmt.Sprintf("unumeric_ff_%v", ffValue)
+ srcFullName := s.attachSchemaSuffix(dstTableName)
+
+ _, err := s.Conn().Exec(context.Background(), fmt.Sprintf(`
+ CREATE TABLE IF NOT EXISTS %s(
+ id INT PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
+ c numeric
+ );
+ `, srcFullName))
+ require.NoError(s.t, err)
+
+ _, err = s.Conn().Exec(context.Background(), fmt.Sprintf("INSERT INTO %s(c) VALUES($1)", srcFullName), nines)
+ require.NoError(s.t, err)
+
+ connectionGen := e2e.FlowConnectionGenerationConfig{
+ FlowJobName: s.attachSuffix(fmt.Sprintf("clickhouse_test_unbounded_numerics_ff_%v", ffValue)),
+ TableNameMapping: map[string]string{srcFullName: dstTableName},
+ Destination: s.Peer().Name,
+ }
+ flowConnConfig := connectionGen.GenerateFlowConnectionConfigs(s.t)
+ flowConnConfig.DoInitialSnapshot = true
+ flowConnConfig.Env = map[string]string{"PEERDB_CLICKHOUSE_UNBOUNDED_NUMERIC_AS_STRING": strconv.FormatBool(ffValue)}
+ tc := e2e.NewTemporalClient(s.t)
+ env := e2e.ExecutePeerflow(tc, peerflow.CDCFlowWorkflow, flowConnConfig, nil)
+ e2e.SetupCDCFlowStatusQuery(s.t, env, flowConnConfig)
+
+ e2e.EnvWaitForCount(env, s, "waiting for CDC count", dstTableName, "id,c", 1)
+
+ _, err = s.Conn().Exec(context.Background(), fmt.Sprintf("INSERT INTO %s(c) VALUES($1)", srcFullName), nines)
+ require.NoError(s.t, err)
+
+ e2e.EnvWaitForCount(env, s, "waiting for CDC count", dstTableName, "id,c", 2)
+
+ rows, err := s.GetRows(dstTableName, "c")
+ require.NoError(s.t, err)
+ require.Len(s.t, rows.Records, 2, "expected 2 rows")
+ for _, row := range rows.Records {
+ require.Len(s.t, row, 1, "expected 1 column")
+ if ffValue {
+ c, ok := row[0].Value().(string)
+ require.True(s.t, ok, "expected unbounded NUMERIC to be String")
+ require.Equal(s.t, nines, c, "expected unbounded NUMERIC to be 9s")
+ } else {
+ c, ok := row[0].Value().(decimal.Decimal)
+ require.True(s.t, ok, "expected unbounded NUMERIC to be Decimal")
+ require.Equal(s.t, nines, c.String(), "expected unbounded NUMERIC to be 9s")
+ }
+ }
+
+ env.Cancel()
+ e2e.RequireEnvCanceled(s.t, env)
+}
+
+func (s ClickHouseSuite) Test_Unbounded_Numeric_With_FF() {
+ s.testNumericFF(true)
+}
+
+func (s ClickHouseSuite) Test_Unbounded_Numeric_Without_FF() {
+ s.testNumericFF(false)
+}
diff --git a/flow/e2e/postgres/postgres.go b/flow/e2e/postgres/postgres.go
index 37a0ace06b..ea43648f7c 100644
--- a/flow/e2e/postgres/postgres.go
+++ b/flow/e2e/postgres/postgres.go
@@ -54,7 +54,10 @@ func (s PeerFlowE2ETestSuitePG) DestinationTable(table string) string {
func (s PeerFlowE2ETestSuitePG) GetRows(table string, cols string) (*model.QRecordBatch, error) {
s.t.Helper()
- pgQueryExecutor := s.conn.NewQRepQueryExecutor("testflow", "testpart")
+ pgQueryExecutor, err := s.conn.NewQRepQueryExecutor(context.Background(), "testflow", "testpart")
+ if err != nil {
+ return nil, err
+ }
return pgQueryExecutor.ExecuteAndProcessQuery(
context.Background(),
diff --git a/flow/e2e/snowflake/snowflake_helper.go b/flow/e2e/snowflake/snowflake_helper.go
index ca57b5b473..7e2943e3bc 100644
--- a/flow/e2e/snowflake/snowflake_helper.go
+++ b/flow/e2e/snowflake/snowflake_helper.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "math/rand/v2"
"os"
"testing"
@@ -13,7 +14,6 @@ import (
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model"
"github.com/PeerDB-io/peer-flow/model/qvalue"
- "github.com/PeerDB-io/peer-flow/shared"
)
type SnowflakeTestHelper struct {
@@ -47,11 +47,8 @@ func NewSnowflakeTestHelper(t *testing.T) (*SnowflakeTestHelper, error) {
return nil, fmt.Errorf("failed to unmarshal json: %w", err)
}
- runID, err := shared.RandomUInt64()
- if err != nil {
- return nil, fmt.Errorf("failed to generate random uint64: %w", err)
- }
-
+ //nolint:gosec // number has no cryptographic significance
+ runID := rand.Uint64()
testDatabaseName := fmt.Sprintf("e2e_test_%d", runID)
adminClient, err := connsnowflake.NewSnowflakeClient(context.Background(), config)
diff --git a/flow/e2e/snowflake/snowflake_schema_delta_test.go b/flow/e2e/snowflake/snowflake_schema_delta_test.go
index 32cb03b644..ada2b10f6a 100644
--- a/flow/e2e/snowflake/snowflake_schema_delta_test.go
+++ b/flow/e2e/snowflake/snowflake_schema_delta_test.go
@@ -53,7 +53,7 @@ func (s SnowflakeSchemaDeltaTestSuite) TestSimpleAddColumn() {
err := s.sfTestHelper.RunCommand(fmt.Sprintf("CREATE TABLE %s(ID TEXT PRIMARY KEY)", tableName))
require.NoError(s.t, err)
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: []*protos.FieldDescription{
@@ -167,7 +167,7 @@ func (s SnowflakeSchemaDeltaTestSuite) TestAddAllColumnTypes() {
}
}
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: addedColumns,
@@ -246,7 +246,7 @@ func (s SnowflakeSchemaDeltaTestSuite) TestAddTrickyColumnNames() {
}
}
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: addedColumns,
@@ -301,7 +301,7 @@ func (s SnowflakeSchemaDeltaTestSuite) TestAddWhitespaceColumnNames() {
}
}
- err = s.connector.ReplayTableSchemaDeltas(context.Background(), "schema_delta_flow", []*protos.TableSchemaDelta{{
+ err = s.connector.ReplayTableSchemaDeltas(context.Background(), nil, "schema_delta_flow", []*protos.TableSchemaDelta{{
SrcTableName: tableName,
DstTableName: tableName,
AddedColumns: addedColumns,
diff --git a/flow/e2e/sqlserver/sqlserver_helper.go b/flow/e2e/sqlserver/sqlserver_helper.go
index 056922800c..d3e1401f24 100644
--- a/flow/e2e/sqlserver/sqlserver_helper.go
+++ b/flow/e2e/sqlserver/sqlserver_helper.go
@@ -3,6 +3,7 @@ package e2e_sqlserver
import (
"context"
"fmt"
+ "math/rand/v2"
"os"
"strconv"
@@ -10,7 +11,6 @@ import (
connsqlserver "github.com/PeerDB-io/peer-flow/connectors/sqlserver"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model/qvalue"
- "github.com/PeerDB-io/peer-flow/shared"
)
type SQLServerHelper struct {
@@ -45,11 +45,8 @@ func NewSQLServerHelper() (*SQLServerHelper, error) {
return nil, fmt.Errorf("invalid connection configs: %v", connErr)
}
- rndNum, err := shared.RandomUInt64()
- if err != nil {
- return nil, err
- }
-
+ //nolint:gosec // number has no cryptographic significance
+ rndNum := rand.Uint64()
testSchema := fmt.Sprintf("e2e_test_%d", rndNum)
if err := connector.CreateSchema(context.Background(), testSchema); err != nil {
return nil, err
diff --git a/flow/e2e/test_utils.go b/flow/e2e/test_utils.go
index ce134f819a..7fb3f857da 100644
--- a/flow/e2e/test_utils.go
+++ b/flow/e2e/test_utils.go
@@ -89,7 +89,10 @@ func EnvTrue(t *testing.T, env WorkflowRun, val bool) {
}
func GetPgRows(conn *connpostgres.PostgresConnector, suffix string, table string, cols string) (*model.QRecordBatch, error) {
- pgQueryExecutor := conn.NewQRepQueryExecutor("testflow", "testpart")
+ pgQueryExecutor, err := conn.NewQRepQueryExecutor(context.Background(), "testflow", "testpart")
+ if err != nil {
+ return nil, err
+ }
return pgQueryExecutor.ExecuteAndProcessQuery(
context.Background(),
@@ -218,7 +221,7 @@ func SetupCDCFlowStatusQuery(t *testing.T, env WorkflowRun, config *protos.FlowC
var status protos.FlowStatus
if err := response.Get(&status); err != nil {
t.Fatal(err)
- } else if status == protos.FlowStatus_STATUS_RUNNING {
+ } else if status == protos.FlowStatus_STATUS_RUNNING || status == protos.FlowStatus_STATUS_COMPLETED {
return
} else if counter > 30 {
env.Cancel()
diff --git a/flow/go.mod b/flow/go.mod
index b7eb9d1d65..a11ffb5a7e 100644
--- a/flow/go.mod
+++ b/flow/go.mod
@@ -53,22 +53,28 @@ require (
github.com/urfave/cli/v3 v3.0.0-alpha9.2
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78
github.com/yuin/gopher-lua v1.1.1
- go.opentelemetry.io/otel v1.31.0
+ go.opentelemetry.io/otel v1.32.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0
- go.opentelemetry.io/otel/metric v1.31.0
- go.opentelemetry.io/otel/sdk v1.31.0
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0
+ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0
+ go.opentelemetry.io/otel/metric v1.32.0
+ go.opentelemetry.io/otel/sdk v1.32.0
go.opentelemetry.io/otel/sdk/metric v1.31.0
+ go.opentelemetry.io/otel/trace v1.32.0
go.temporal.io/api v1.41.0
go.temporal.io/sdk v1.30.0
go.temporal.io/sdk/contrib/opentelemetry v0.6.0
go.uber.org/automaxprocs v1.6.0
golang.org/x/crypto v0.28.0
- golang.org/x/sync v0.8.0
+ golang.org/x/sync v0.9.0
google.golang.org/api v0.204.0
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28
google.golang.org/grpc v1.67.1
google.golang.org/protobuf v1.35.1
+ k8s.io/apimachinery v0.31.2
+ k8s.io/client-go v0.31.2
)
require (
@@ -105,18 +111,29 @@ require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
github.com/dvsekhvalnov/jose2go v1.7.0 // indirect
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
+ github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/envoyproxy/go-control-plane v0.13.1 // indirect
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
+ github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
github.com/getsentry/sentry-go v0.29.1 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/go-openapi/jsonpointer v0.19.6 // indirect
+ github.com/go-openapi/jsonreference v0.20.2 // indirect
+ github.com/go-openapi/swag v0.22.4 // indirect
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
+ github.com/google/gnostic-models v0.6.8 // indirect
+ github.com/google/go-cmp v0.6.0 // indirect
+ github.com/google/gofuzz v1.2.0 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
+ github.com/josharian/intern v1.0.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
@@ -124,6 +141,9 @@ require (
github.com/lestrrat-go/httprc v1.0.6 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
+ github.com/mailru/easyjson v0.7.7 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nexus-rpc/sdk-go v0.0.11 // indirect
@@ -138,14 +158,23 @@ require (
github.com/segmentio/asm v1.2.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/twmb/franz-go/pkg/kmsg v1.9.0 // indirect
+ github.com/x448/float16 v0.8.4 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.31.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
- go.opentelemetry.io/otel/trace v1.31.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/term v0.25.0 // indirect
google.golang.org/grpc/stats/opentelemetry v0.0.0-20241028142157-ada6787961b3 // indirect
+ gopkg.in/inf.v0 v0.9.1 // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+ k8s.io/api v0.31.2 // indirect
+ k8s.io/klog/v2 v2.130.1 // indirect
+ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+ k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
+ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
+ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+ sigs.k8s.io/yaml v1.4.0 // indirect
)
require (
@@ -165,7 +194,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.3 // indirect
- github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/djherbis/buffer v1.2.0
github.com/djherbis/nio/v3 v3.0.1
github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect
@@ -193,7 +222,7 @@ require (
github.com/pborman/uuid v1.2.1 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
@@ -201,8 +230,8 @@ require (
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
golang.org/x/net v0.30.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
- golang.org/x/sys v0.26.0 // indirect
- golang.org/x/text v0.19.0 // indirect
+ golang.org/x/sys v0.27.0 // indirect
+ golang.org/x/text v0.20.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/tools v0.26.0 // indirect
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
diff --git a/flow/go.sum b/flow/go.sum
index 71299452d2..7a0380da03 100644
--- a/flow/go.sum
+++ b/flow/go.sum
@@ -180,8 +180,9 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0=
github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
@@ -197,6 +198,8 @@ github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHo
github.com/elastic/elastic-transport-go/v8 v8.6.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
github.com/elastic/go-elasticsearch/v8 v8.15.0 h1:IZyJhe7t7WI3NEFdcHnf6IJXqpRf+8S8QWLtZYYyBYk=
github.com/elastic/go-elasticsearch/v8 v8.15.0/go.mod h1:HCON3zj4btpqs2N1jjsAy4a/fiAul+YBP00mBH4xik8=
+github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
+github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -211,6 +214,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
+github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
+github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
github.com/getsentry/sentry-go v0.29.1 h1:DyZuChN8Hz3ARxGVV8ePaNXh1dQ7d76AiB117xcREwA=
@@ -228,9 +233,18 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
+github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
+github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
@@ -270,6 +284,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI=
github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
+github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -279,10 +295,16 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
+github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
+github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM=
+github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -330,6 +352,10 @@ github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs=
github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
@@ -341,6 +367,7 @@ github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -365,10 +392,17 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/linkedin/goavro/v2 v2.13.0 h1:L8eI8GcuciwUkt41Ej62joSZS4kKaYIUdze+6for9NU=
github.com/linkedin/goavro/v2 v2.13.0/go.mod h1:KXx+erlq+RPlGSPmLF7xGo6SAbh8sCQ53x064+ioxhk=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/microsoft/go-mssqldb v1.7.2 h1:CHkFJiObW7ItKTJfHo1QX7QBBD1iV+mn1eOyRP3b/PA=
github.com/microsoft/go-mssqldb v1.7.2/go.mod h1:kOvZKUdrhhFQmxLZqbwUV0rHkNkZpthMITIb2Ko1IoA=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
@@ -377,6 +411,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
github.com/nexus-rpc/sdk-go v0.0.11 h1:qH3Us3spfp50t5ca775V1va2eE6z1zMQDZY4mvbw0CI=
github.com/nexus-rpc/sdk-go v0.0.11/go.mod h1:TpfkM2Cw0Rlk9drGkoiSMpFqflKTiQLWUNyKJjF8mKQ=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA=
+github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To=
+github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
+github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c=
github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM=
@@ -397,8 +435,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y=
@@ -428,6 +467,8 @@ github.com/slack-go/slack v0.15.0 h1:LE2lj2y9vqqiOf+qIIy0GvEoxgF1N5yLGZffmEZykt0
github.com/slack-go/slack v0.15.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
github.com/snowflakedb/gosnowflake v1.12.0 h1:Saez8egtn5xAoVMBxFaMu9MYfAG9SS9dpAEXD1/ECIo=
github.com/snowflakedb/gosnowflake v1.12.0/go.mod h1:wHfYmZi3zvtWItojesAhWWXBN7+niex2R1h/S7QCZYg=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
@@ -458,6 +499,8 @@ github.com/twpayne/go-geos v0.19.0 h1:V7vnLe7gY7JOHLTg8+2oykZOw6wpBLHVNlcnzS2FlG
github.com/twpayne/go-geos v0.19.0/go.mod h1:XGpUjCtZf4Ul6BMii6KA4EmJ9JCNhVP1mohdoReopZ8=
github.com/urfave/cli/v3 v3.0.0-alpha9.2 h1:CL8llQj3dGRLVQQzHxS+ZYRLanOuhyK1fXgLKD+qV+Y=
github.com/urfave/cli/v3 v3.0.0-alpha9.2/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y=
+github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
+github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
@@ -486,20 +529,26 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.5
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.56.0/go.mod h1:n8MR6/liuGB5EmTETUBeU5ZgqMOlqKRxUaqPQBOANZ8=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 h1:UP6IpuHFkUgOQL9FFQFrZ+5LiwhhYRbi7VZSIx6Nj5s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0/go.mod h1:qxuZLtbq5QDtdeSHsS7bcf6EH6uO6jUAgk764zd3rhM=
-go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
-go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
+go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
+go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 h1:FZ6ei8GFW7kyPYdxJaV2rgI6M+4tvZzhYsQ2wgyVC08=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0/go.mod h1:MdEu/mC6j3D+tTEfvI15b5Ci2Fn7NneJ71YMoiS3tpI=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0/go.mod h1:hg1zaDMpyZJuUzjFxFsRYBoccE86tM9Uf4IqNMUxvrY=
-go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
-go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
-go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk=
-go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 h1:IJFEoHiytixx8cMiVAO+GmHR6Frwu+u5Ur8njpFO6Ac=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0/go.mod h1:3rHrKNtLIoS0oZwkY2vxi+oJcwFRWdtUyRII+so45p8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 h1:9kV11HXBHZAvuPUZxmMWrH8hZn/6UnHX4K0mu36vNsU=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0/go.mod h1:JyA0FHXe22E1NeNiHmVp7kFHglnexDQ7uRWDiiJ1hKQ=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
+go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
+go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
+go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
+go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU=
go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc=
go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8=
-go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
-go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
+go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
+go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.temporal.io/api v1.41.0 h1:VYzyWJjJk1jeB9urntA/t7Hiyo2tHdM5xEdtdib4EO8=
@@ -512,6 +561,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -554,8 +605,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
-golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -570,8 +621,8 @@ golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
-golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
@@ -579,8 +630,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
-golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
+golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -645,6 +696,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
@@ -655,5 +708,23 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.31.2 h1:3wLBbL5Uom/8Zy98GRPXpJ254nEFpl+hwndmk9RwmL0=
+k8s.io/api v0.31.2/go.mod h1:bWmGvrGPssSK1ljmLzd3pwCQ9MgoTsRCuK35u6SygUk=
+k8s.io/apimachinery v0.31.2 h1:i4vUt2hPK56W6mlT7Ry+AO8eEsyxMD1U44NR22CLTYw=
+k8s.io/apimachinery v0.31.2/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/client-go v0.31.2 h1:Y2F4dxU5d3AQj+ybwSMqQnpZH9F30//1ObxOKlTI9yc=
+k8s.io/client-go v0.31.2/go.mod h1:NPa74jSVR/+eez2dFsEIHNa+3o09vtNaWwWwb1qSxSs=
+k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=
+k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag=
+k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A=
+k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
nhooyr.io/websocket v1.8.11 h1:f/qXNc2/3DpoSZkHt1DQu6rj4zGC8JmkkLkWss0MgN0=
nhooyr.io/websocket v1.8.11/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
+sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
+sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
+sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
+sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
diff --git a/flow/main.go b/flow/main.go
index 4001a88912..613c426340 100644
--- a/flow/main.go
+++ b/flow/main.go
@@ -70,6 +70,60 @@ func main() {
Sources: cli.EnvVars("TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASKS"),
}
+ maintenanceModeWorkflowFlag := &cli.StringFlag{
+ Name: "run-maintenance-flow",
+ Value: "",
+ Usage: "Run a maintenance flow. Options are 'start' or 'end'",
+ Sources: cli.EnvVars("RUN_MAINTENANCE_FLOW"),
+ }
+
+ maintenanceSkipOnApiVersionMatchFlag := &cli.BoolFlag{
+ Name: "skip-on-api-version-match",
+ Value: false,
+ Usage: "Skip maintenance flow if the API version matches",
+ Sources: cli.EnvVars("MAINTENANCE_SKIP_ON_API_VERSION_MATCH"),
+ }
+
+ maintenanceSkipOnNoMirrorsFlag := &cli.BoolFlag{
+ Name: "skip-on-no-mirrors",
+ Value: false,
+ Usage: "Skip maintenance flow if there are no mirrors",
+ Sources: cli.EnvVars("MAINTENANCE_SKIP_ON_NO_MIRRORS"),
+ }
+
+ flowGrpcAddressFlag := &cli.StringFlag{
+ Name: "flow-grpc-address",
+ Value: "",
+ Usage: "Address of the flow gRPC server",
+ Sources: cli.EnvVars("FLOW_GRPC_ADDRESS"),
+ }
+
+ flowTlsEnabledFlag := &cli.BoolFlag{
+ Name: "flow-tls-enabled",
+ Value: false,
+ Usage: "Enable TLS for the flow gRPC server",
+ Sources: cli.EnvVars("FLOW_TLS_ENABLED"),
+ }
+
+ useMaintenanceTaskQueueFlag := &cli.BoolFlag{
+ Name: "use-maintenance-task-queue",
+ Value: false,
+ Usage: "Use the maintenance task queue for the worker",
+ Sources: cli.EnvVars("USE_MAINTENANCE_TASK_QUEUE"),
+ }
+
+ assumedSkippedMaintenanceWorkflowsFlag := &cli.BoolFlag{
+ Name: "assume-skipped-workflow",
+ Value: false,
+ Usage: "Skip running maintenance workflows and simply output to catalog",
+ }
+
+ skipIfK8sServiceMissingFlag := &cli.StringFlag{
+ Name: "skip-if-k8s-service-missing",
+ Value: "",
+ Usage: "Skip maintenance if the k8s service is missing, generally used during pre-upgrade hook",
+ }
+
app := &cli.Command{
Name: "PeerDB Flows CLI",
Commands: []*cli.Command{
@@ -85,11 +139,12 @@ func main() {
TemporalNamespace: clicmd.String("temporal-namespace"),
TemporalMaxConcurrentActivities: int(clicmd.Int("temporal-max-concurrent-activities")),
TemporalMaxConcurrentWorkflowTasks: int(clicmd.Int("temporal-max-concurrent-workflow-tasks")),
+ UseMaintenanceTaskQueue: clicmd.Bool(useMaintenanceTaskQueueFlag.Name),
})
if err != nil {
return err
}
- defer res.Cleanup()
+ defer res.Close()
return res.Worker.Run(worker.InterruptCh())
},
Flags: []cli.Flag{
@@ -100,6 +155,7 @@ func main() {
temporalNamespaceFlag,
temporalMaxConcurrentActivitiesFlag,
temporalMaxConcurrentWorkflowTasksFlag,
+ useMaintenanceTaskQueueFlag,
},
},
{
@@ -148,6 +204,37 @@ func main() {
})
},
},
+ {
+ Name: "maintenance",
+ Flags: []cli.Flag{
+ temporalHostPortFlag,
+ temporalNamespaceFlag,
+ maintenanceModeWorkflowFlag,
+ maintenanceSkipOnApiVersionMatchFlag,
+ maintenanceSkipOnNoMirrorsFlag,
+ flowGrpcAddressFlag,
+ flowTlsEnabledFlag,
+ useMaintenanceTaskQueueFlag,
+ assumedSkippedMaintenanceWorkflowsFlag,
+ skipIfK8sServiceMissingFlag,
+ },
+ Action: func(ctx context.Context, clicmd *cli.Command) error {
+ temporalHostPort := clicmd.String("temporal-host-port")
+
+ return cmd.MaintenanceMain(ctx, &cmd.MaintenanceCLIParams{
+ TemporalHostPort: temporalHostPort,
+ TemporalNamespace: clicmd.String(temporalNamespaceFlag.Name),
+ Mode: clicmd.String(maintenanceModeWorkflowFlag.Name),
+ SkipOnApiVersionMatch: clicmd.Bool(maintenanceSkipOnApiVersionMatchFlag.Name),
+ SkipOnNoMirrors: clicmd.Bool(maintenanceSkipOnNoMirrorsFlag.Name),
+ FlowGrpcAddress: clicmd.String(flowGrpcAddressFlag.Name),
+ FlowTlsEnabled: clicmd.Bool(flowTlsEnabledFlag.Name),
+ UseMaintenanceTaskQueue: clicmd.Bool(useMaintenanceTaskQueueFlag.Name),
+ AssumeSkippedMaintenanceWorkflows: clicmd.Bool(assumedSkippedMaintenanceWorkflowsFlag.Name),
+ SkipIfK8sServiceMissing: clicmd.String(skipIfK8sServiceMissingFlag.Name),
+ })
+ },
+ },
},
}
@@ -164,5 +251,6 @@ func main() {
if err := app.Run(appCtx, os.Args); err != nil {
log.Printf("error running app: %+v", err)
+ panic(err)
}
}
diff --git a/flow/model/conversion_avro.go b/flow/model/conversion_avro.go
index 8f52c44611..ec7cfc6e37 100644
--- a/flow/model/conversion_avro.go
+++ b/flow/model/conversion_avro.go
@@ -1,6 +1,7 @@
package model
import (
+ "context"
"encoding/json"
"fmt"
@@ -8,38 +9,52 @@ import (
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/model/qvalue"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
)
type QRecordAvroConverter struct {
- logger log.Logger
- Schema *QRecordAvroSchemaDefinition
- ColNames []string
- TargetDWH protos.DBType
+ logger log.Logger
+ Schema *QRecordAvroSchemaDefinition
+ ColNames []string
+ TargetDWH protos.DBType
+ UnboundedNumericAsString bool
}
func NewQRecordAvroConverter(
+ ctx context.Context,
+ env map[string]string,
schema *QRecordAvroSchemaDefinition,
targetDWH protos.DBType,
colNames []string,
logger log.Logger,
-) *QRecordAvroConverter {
- return &QRecordAvroConverter{
- Schema: schema,
- TargetDWH: targetDWH,
- ColNames: colNames,
- logger: logger,
+) (*QRecordAvroConverter, error) {
+ var unboundedNumericAsString bool
+ if targetDWH == protos.DBType_CLICKHOUSE {
+ var err error
+ unboundedNumericAsString, err = peerdbenv.PeerDBEnableClickHouseNumericAsString(ctx, env)
+ if err != nil {
+ return nil, err
+ }
}
-}
-func (qac *QRecordAvroConverter) Convert(qrecord []qvalue.QValue) (map[string]interface{}, error) {
- m := make(map[string]interface{}, len(qrecord))
+ return &QRecordAvroConverter{
+ Schema: schema,
+ TargetDWH: targetDWH,
+ ColNames: colNames,
+ logger: logger,
+ UnboundedNumericAsString: unboundedNumericAsString,
+ }, nil
+}
+func (qac *QRecordAvroConverter) Convert(qrecord []qvalue.QValue) (map[string]any, error) {
+ m := make(map[string]any, len(qrecord))
for idx, val := range qrecord {
avroVal, err := qvalue.QValueToAvro(
val,
&qac.Schema.Fields[idx],
qac.TargetDWH,
qac.logger,
+ qac.UnboundedNumericAsString,
)
if err != nil {
return nil, fmt.Errorf("failed to convert QValue to Avro-compatible value: %w", err)
@@ -52,8 +67,8 @@ func (qac *QRecordAvroConverter) Convert(qrecord []qvalue.QValue) (map[string]in
}
type QRecordAvroField struct {
- Type interface{} `json:"type"`
- Name string `json:"name"`
+ Type any `json:"type"`
+ Name string `json:"name"`
}
type QRecordAvroSchema struct {
@@ -68,6 +83,8 @@ type QRecordAvroSchemaDefinition struct {
}
func GetAvroSchemaDefinition(
+ ctx context.Context,
+ env map[string]string,
dstTableName string,
qRecordSchema qvalue.QRecordSchema,
targetDWH protos.DBType,
@@ -75,7 +92,7 @@ func GetAvroSchemaDefinition(
avroFields := make([]QRecordAvroField, 0, len(qRecordSchema.Fields))
for _, qField := range qRecordSchema.Fields {
- avroType, err := qvalue.GetAvroSchemaFromQValueKind(qField.Type, targetDWH, qField.Precision, qField.Scale)
+ avroType, err := qvalue.GetAvroSchemaFromQValueKind(ctx, env, qField.Type, targetDWH, qField.Precision, qField.Scale)
if err != nil {
return nil, err
}
diff --git a/flow/model/qrecord_copy_from_source.go b/flow/model/qrecord_copy_from_source.go
index 308676c5f5..d633fda999 100644
--- a/flow/model/qrecord_copy_from_source.go
+++ b/flow/model/qrecord_copy_from_source.go
@@ -1,6 +1,7 @@
package model
import (
+ "encoding/json"
"errors"
"fmt"
"strings"
@@ -82,6 +83,8 @@ func (src *QRecordCopyFromSource) Values() ([]interface{}, error) {
values[i] = str
case qvalue.QValueTime:
values[i] = pgtype.Time{Microseconds: v.Val.UnixMicro(), Valid: true}
+ case qvalue.QValueTSTZRange:
+ values[i] = v.Val
case qvalue.QValueTimestamp:
values[i] = pgtype.Timestamp{Time: v.Val, Valid: true}
case qvalue.QValueTimestampTZ:
@@ -170,8 +173,16 @@ func (src *QRecordCopyFromSource) Values() ([]interface{}, error) {
}
values[i] = a
case qvalue.QValueJSON:
- values[i] = v.Val
+ if v.IsArray {
+ var arrayJ []interface{}
+ if err := json.Unmarshal([]byte(v.Value().(string)), &arrayJ); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal JSON array: %v", err)
+ }
+ values[i] = arrayJ
+ } else {
+ values[i] = v.Value()
+ }
// And so on for the other types...
default:
return nil, fmt.Errorf("unsupported value type %T", qValue)
diff --git a/flow/model/qrecord_stream.go b/flow/model/qrecord_stream.go
index 3bb2d1f248..054d6a42b1 100644
--- a/flow/model/qrecord_stream.go
+++ b/flow/model/qrecord_stream.go
@@ -30,8 +30,8 @@ func (s *QRecordStream) Schema() qvalue.QRecordSchema {
func (s *QRecordStream) SetSchema(schema qvalue.QRecordSchema) {
if !s.schemaSet {
s.schema = schema
- close(s.schemaLatch)
s.schemaSet = true
+ close(s.schemaLatch)
}
}
diff --git a/flow/model/qvalue/avro_converter.go b/flow/model/qvalue/avro_converter.go
index 9738f46e8f..db5bf4e2af 100644
--- a/flow/model/qvalue/avro_converter.go
+++ b/flow/model/qvalue/avro_converter.go
@@ -1,6 +1,7 @@
package qvalue
import (
+ "context"
"encoding/base64"
"errors"
"fmt"
@@ -14,6 +15,7 @@ import (
"github.com/PeerDB-io/peer-flow/datatypes"
"github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
)
type AvroSchemaField struct {
@@ -74,7 +76,14 @@ func TruncateOrLogNumeric(num decimal.Decimal, precision int16, scale int16, tar
//
// For example, QValueKindInt64 would return an AvroLogicalSchema of "long". Unsupported QValueKinds
// will return an error.
-func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, precision int16, scale int16) (interface{}, error) {
+func GetAvroSchemaFromQValueKind(
+ ctx context.Context,
+ env map[string]string,
+ kind QValueKind,
+ targetDWH protos.DBType,
+ precision int16,
+ scale int16,
+) (interface{}, error) {
switch kind {
case QValueKindString:
return "string", nil
@@ -103,9 +112,19 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci
}
return "bytes", nil
case QValueKindNumeric:
- if targetDWH == protos.DBType_CLICKHOUSE &&
- precision > datatypes.PeerDBClickHouseMaxPrecision {
- return "string", nil
+ if targetDWH == protos.DBType_CLICKHOUSE {
+ if precision == 0 && scale == 0 {
+ asString, err := peerdbenv.PeerDBEnableClickHouseNumericAsString(ctx, env)
+ if err != nil {
+ return nil, err
+ }
+ if asString {
+ return "string", nil
+ }
+ }
+ if precision > datatypes.PeerDBClickHouseMaxPrecision {
+ return "string", nil
+ }
}
avroNumericPrecision, avroNumericScale := DetermineNumericSettingForDWH(precision, scale, targetDWH)
return AvroSchemaNumeric{
@@ -138,7 +157,9 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci
}, nil
}
return "string", nil
- case QValueKindHStore, QValueKindJSON, QValueKindStruct:
+ case QValueKindTSTZRange:
+ return "string", nil
+ case QValueKindHStore, QValueKindJSON, QValueKindJSONB, QValueKindStruct:
return "string", nil
case QValueKindArrayFloat32:
return AvroSchemaArray{
@@ -193,6 +214,8 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci
Type: "array",
Items: "string",
}, nil
+ case QValueKindArrayJSON, QValueKindArrayJSONB:
+ return "string", nil
case QValueKindArrayString:
return AvroSchemaArray{
Type: "array",
@@ -208,19 +231,24 @@ func GetAvroSchemaFromQValueKind(kind QValueKind, targetDWH protos.DBType, preci
type QValueAvroConverter struct {
*QField
- logger log.Logger
- TargetDWH protos.DBType
+ logger log.Logger
+ TargetDWH protos.DBType
+ UnboundedNumericAsString bool
}
-func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger log.Logger) (interface{}, error) {
+func QValueToAvro(
+ value QValue, field *QField, targetDWH protos.DBType, logger log.Logger,
+ unboundedNumericAsString bool,
+) (any, error) {
if value.Value() == nil {
return nil, nil
}
- c := &QValueAvroConverter{
- QField: field,
- TargetDWH: targetDWH,
- logger: logger,
+ c := QValueAvroConverter{
+ QField: field,
+ TargetDWH: targetDWH,
+ logger: logger,
+ UnboundedNumericAsString: unboundedNumericAsString,
}
switch v := value.(type) {
@@ -315,7 +343,7 @@ func QValueToAvro(value QValue, field *QField, targetDWH protos.DBType, logger l
return t, nil
case QValueQChar:
return c.processNullableUnion("string", string(v.Val))
- case QValueString, QValueCIDR, QValueINET, QValueMacaddr, QValueInterval:
+ case QValueString, QValueCIDR, QValueINET, QValueMacaddr, QValueInterval, QValueTSTZRange:
if c.TargetDWH == protos.DBType_SNOWFLAKE && v.Value() != nil &&
(len(v.Value().(string)) > 15*1024*1024) {
slog.Warn("Clearing TEXT value > 15MB for Snowflake!")
@@ -452,18 +480,18 @@ func (c *QValueAvroConverter) processNullableUnion(
return value, nil
}
-func (c *QValueAvroConverter) processNumeric(num decimal.Decimal) interface{} {
+func (c *QValueAvroConverter) processNumeric(num decimal.Decimal) any {
+ if (c.UnboundedNumericAsString && c.Precision == 0 && c.Scale == 0) ||
+ (c.TargetDWH == protos.DBType_CLICKHOUSE && c.Precision > datatypes.PeerDBClickHouseMaxPrecision) {
+ numStr, _ := c.processNullableUnion("string", num.String())
+ return numStr
+ }
+
num, err := TruncateOrLogNumeric(num, c.Precision, c.Scale, c.TargetDWH)
if err != nil {
return nil
}
- if c.TargetDWH == protos.DBType_CLICKHOUSE &&
- c.Precision > datatypes.PeerDBClickHouseMaxPrecision {
- // no error returned
- numStr, _ := c.processNullableUnion("string", num.String())
- return numStr
- }
rat := num.Rat()
if c.Nullable {
return goavro.Union("bytes.decimal", rat)
diff --git a/flow/model/qvalue/dwh.go b/flow/model/qvalue/dwh.go
index 49c359b885..b2d085acb4 100644
--- a/flow/model/qvalue/dwh.go
+++ b/flow/model/qvalue/dwh.go
@@ -5,24 +5,24 @@ import (
"go.temporal.io/sdk/log"
- numeric "github.com/PeerDB-io/peer-flow/datatypes"
+ "github.com/PeerDB-io/peer-flow/datatypes"
"github.com/PeerDB-io/peer-flow/generated/protos"
)
func DetermineNumericSettingForDWH(precision int16, scale int16, dwh protos.DBType) (int16, int16) {
- var warehouseNumeric numeric.WarehouseNumericCompatibility
+ var warehouseNumeric datatypes.WarehouseNumericCompatibility
switch dwh {
case protos.DBType_CLICKHOUSE:
- warehouseNumeric = numeric.ClickHouseNumericCompatibility{}
+ warehouseNumeric = datatypes.ClickHouseNumericCompatibility{}
case protos.DBType_SNOWFLAKE:
- warehouseNumeric = numeric.SnowflakeNumericCompatibility{}
+ warehouseNumeric = datatypes.SnowflakeNumericCompatibility{}
case protos.DBType_BIGQUERY:
- warehouseNumeric = numeric.BigQueryNumericCompatibility{}
+ warehouseNumeric = datatypes.BigQueryNumericCompatibility{}
default:
- warehouseNumeric = numeric.DefaultNumericCompatibility{}
+ warehouseNumeric = datatypes.DefaultNumericCompatibility{}
}
- return numeric.GetNumericTypeForWarehouse(numeric.MakeNumericTypmod(int32(precision), int32(scale)), warehouseNumeric)
+ return datatypes.GetNumericTypeForWarehousePrecisionScale(precision, scale, warehouseNumeric)
}
// Bigquery will not allow timestamp if it is less than 1AD and more than 9999AD
diff --git a/flow/model/qvalue/kind.go b/flow/model/qvalue/kind.go
index 79e8f89e40..3cffcc274a 100644
--- a/flow/model/qvalue/kind.go
+++ b/flow/model/qvalue/kind.go
@@ -1,10 +1,13 @@
package qvalue
import (
+ "context"
"fmt"
"strings"
+ "github.com/PeerDB-io/peer-flow/datatypes"
"github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
)
type QValueKind string
@@ -26,10 +29,12 @@ const (
QValueKindTime QValueKind = "time"
QValueKindTimeTZ QValueKind = "timetz"
QValueKindInterval QValueKind = "interval"
+ QValueKindTSTZRange QValueKind = "tstzrange"
QValueKindNumeric QValueKind = "numeric"
QValueKindBytes QValueKind = "bytes"
QValueKindUUID QValueKind = "uuid"
QValueKindJSON QValueKind = "json"
+ QValueKindJSONB QValueKind = "jsonb"
QValueKindHStore QValueKind = "hstore"
QValueKindGeography QValueKind = "geography"
QValueKindGeometry QValueKind = "geometry"
@@ -51,6 +56,8 @@ const (
QValueKindArrayTimestamp QValueKind = "array_timestamp"
QValueKindArrayTimestampTZ QValueKind = "array_timestamptz"
QValueKindArrayBoolean QValueKind = "array_bool"
+ QValueKindArrayJSON QValueKind = "array_json"
+ QValueKindArrayJSONB QValueKind = "array_jsonb"
)
func (kind QValueKind) IsArray() bool {
@@ -64,10 +71,10 @@ var QValueKindToSnowflakeTypeMap = map[QValueKind]string{
QValueKindInt64: "INTEGER",
QValueKindFloat32: "FLOAT",
QValueKindFloat64: "FLOAT",
- QValueKindNumeric: "NUMBER(38, 9)",
QValueKindQChar: "CHAR",
QValueKindString: "STRING",
QValueKindJSON: "VARIANT",
+ QValueKindJSONB: "VARIANT",
QValueKindTimestamp: "TIMESTAMP_NTZ",
QValueKindTimestampTZ: "TIMESTAMP_TZ",
QValueKindInterval: "VARIANT",
@@ -94,6 +101,8 @@ var QValueKindToSnowflakeTypeMap = map[QValueKind]string{
QValueKindArrayTimestamp: "VARIANT",
QValueKindArrayTimestampTZ: "VARIANT",
QValueKindArrayBoolean: "VARIANT",
+ QValueKindArrayJSON: "VARIANT",
+ QValueKindArrayJSONB: "VARIANT",
}
var QValueKindToClickHouseTypeMap = map[QValueKind]string{
@@ -103,12 +112,12 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{
QValueKindInt64: "Int64",
QValueKindFloat32: "Float32",
QValueKindFloat64: "Float64",
- QValueKindNumeric: "Decimal128(9)",
QValueKindQChar: "FixedString(1)",
QValueKindString: "String",
QValueKindJSON: "String",
QValueKindTimestamp: "DateTime64(6)",
QValueKindTimestampTZ: "DateTime64(6)",
+ QValueKindTSTZRange: "String",
QValueKindTime: "DateTime64(6)",
QValueKindTimeTZ: "DateTime64(6)",
QValueKindDate: "Date32",
@@ -118,7 +127,6 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{
QValueKindInvalid: "String",
QValueKindHStore: "String",
- // array types will be mapped to VARIANT
QValueKindArrayFloat32: "Array(Float32)",
QValueKindArrayFloat64: "Array(Float64)",
QValueKindArrayInt32: "Array(Int32)",
@@ -129,18 +137,43 @@ var QValueKindToClickHouseTypeMap = map[QValueKind]string{
QValueKindArrayDate: "Array(Date)",
QValueKindArrayTimestamp: "Array(DateTime64(6))",
QValueKindArrayTimestampTZ: "Array(DateTime64(6))",
+ QValueKindArrayJSON: "String",
+ QValueKindArrayJSONB: "String",
+}
+
+func getClickHouseTypeForNumericColumn(ctx context.Context, env map[string]string, column *protos.FieldDescription) (string, error) {
+ if column.TypeModifier == -1 {
+ numericAsStringEnabled, err := peerdbenv.PeerDBEnableClickHouseNumericAsString(ctx, env)
+ if err != nil {
+ return "", err
+ }
+ if numericAsStringEnabled {
+ return "String", nil
+ }
+ } else if rawPrecision, _ := datatypes.ParseNumericTypmod(column.TypeModifier); rawPrecision > datatypes.PeerDBClickHouseMaxPrecision {
+ return "String", nil
+ }
+ precision, scale := datatypes.GetNumericTypeForWarehouse(column.TypeModifier, datatypes.ClickHouseNumericCompatibility{})
+ return fmt.Sprintf("Decimal(%d, %d)", precision, scale), nil
}
-func (kind QValueKind) ToDWHColumnType(dwhType protos.DBType) (string, error) {
+// SEE ALSO: QField ToDWHColumnType
+func (kind QValueKind) ToDWHColumnType(ctx context.Context, env map[string]string, dwhType protos.DBType, column *protos.FieldDescription,
+) (string, error) {
switch dwhType {
case protos.DBType_SNOWFLAKE:
- if val, ok := QValueKindToSnowflakeTypeMap[kind]; ok {
+ if kind == QValueKindNumeric {
+ precision, scale := datatypes.GetNumericTypeForWarehouse(column.TypeModifier, datatypes.SnowflakeNumericCompatibility{})
+ return fmt.Sprintf("NUMERIC(%d,%d)", precision, scale), nil
+ } else if val, ok := QValueKindToSnowflakeTypeMap[kind]; ok {
return val, nil
} else {
return "STRING", nil
}
case protos.DBType_CLICKHOUSE:
- if val, ok := QValueKindToClickHouseTypeMap[kind]; ok {
+ if kind == QValueKindNumeric {
+ return getClickHouseTypeForNumericColumn(ctx, env, column)
+ } else if val, ok := QValueKindToClickHouseTypeMap[kind]; ok {
return val, nil
} else {
return "String", nil
diff --git a/flow/model/qvalue/qschema.go b/flow/model/qvalue/qschema.go
index a956968ac1..a6632fdf5f 100644
--- a/flow/model/qvalue/qschema.go
+++ b/flow/model/qvalue/qschema.go
@@ -1,7 +1,13 @@
package qvalue
import (
+ "context"
+ "fmt"
"strings"
+
+ "github.com/PeerDB-io/peer-flow/datatypes"
+ "github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
)
type QField struct {
@@ -47,3 +53,42 @@ func (q QRecordSchema) GetColumnNames() []string {
}
return names
}
+
+func (q QField) getClickHouseTypeForNumericField(ctx context.Context, env map[string]string) (string, error) {
+ if q.Precision == 0 && q.Scale == 0 {
+ numericAsStringEnabled, err := peerdbenv.PeerDBEnableClickHouseNumericAsString(ctx, env)
+ if err != nil {
+ return "", err
+ }
+ if numericAsStringEnabled {
+ return "String", nil
+ }
+ } else if q.Precision > datatypes.PeerDBClickHouseMaxPrecision {
+ return "String", nil
+ }
+ return fmt.Sprintf("Decimal(%d, %d)", q.Precision, q.Scale), nil
+}
+
+// SEE ALSO: qvalue/kind.go ToDWHColumnType
+func (q QField) ToDWHColumnType(ctx context.Context, env map[string]string, dwhType protos.DBType) (string, error) {
+ switch dwhType {
+ case protos.DBType_SNOWFLAKE:
+ if val, ok := QValueKindToSnowflakeTypeMap[q.Type]; ok {
+ return val, nil
+ } else if q.Type == QValueKindNumeric {
+ return fmt.Sprintf("NUMERIC(%d,%d)", q.Precision, q.Scale), nil
+ } else {
+ return "STRING", nil
+ }
+ case protos.DBType_CLICKHOUSE:
+ if val, ok := QValueKindToClickHouseTypeMap[q.Type]; ok {
+ return q.getClickHouseTypeForNumericField(ctx, env)
+ } else if q.Type == QValueKindNumeric {
+ return val, nil
+ } else {
+ return "String", nil
+ }
+ default:
+ return "", fmt.Errorf("unknown dwh type: %v", dwhType)
+ }
+}
diff --git a/flow/model/qvalue/qvalue.go b/flow/model/qvalue/qvalue.go
index 9b1c13f755..1277881a3d 100644
--- a/flow/model/qvalue/qvalue.go
+++ b/flow/model/qvalue/qvalue.go
@@ -6,7 +6,7 @@ import (
"github.com/google/uuid"
"github.com/shopspring/decimal"
- "github.com/yuin/gopher-lua"
+ lua "github.com/yuin/gopher-lua"
"github.com/PeerDB-io/glua64"
"github.com/PeerDB-io/peer-flow/shared"
@@ -294,6 +294,22 @@ func (v QValueInterval) LValue(ls *lua.LState) lua.LValue {
return lua.LString(v.Val)
}
+type QValueTSTZRange struct {
+ Val string
+}
+
+func (QValueTSTZRange) Kind() QValueKind {
+ return QValueKindInterval
+}
+
+func (v QValueTSTZRange) Value() any {
+ return v.Val
+}
+
+func (v QValueTSTZRange) LValue(ls *lua.LState) lua.LValue {
+ return lua.LString(v.Val)
+}
+
type QValueNumeric struct {
Val decimal.Decimal
}
@@ -343,7 +359,8 @@ func (v QValueUUID) LValue(ls *lua.LState) lua.LValue {
}
type QValueJSON struct {
- Val string
+ Val string
+ IsArray bool
}
func (QValueJSON) Kind() QValueKind {
diff --git a/flow/otel_metrics/env.go b/flow/otel_metrics/env.go
deleted file mode 100644
index 81b5d0c3ea..0000000000
--- a/flow/otel_metrics/env.go
+++ /dev/null
@@ -1,11 +0,0 @@
-package otel_metrics
-
-import "github.com/PeerDB-io/peer-flow/peerdbenv"
-
-func GetPeerDBOtelMetricsNamespace() string {
- return peerdbenv.GetEnvString("PEERDB_OTEL_METRICS_NAMESPACE", "")
-}
-
-func GetPeerDBOtelTemporalMetricsExportListEnv() string {
- return peerdbenv.GetEnvString("PEERDB_OTEL_TEMPORAL_METRICS_EXPORT_LIST", "")
-}
diff --git a/flow/otel_metrics/otel_manager.go b/flow/otel_metrics/otel_manager.go
index becf13a16f..dc3deb4246 100644
--- a/flow/otel_metrics/otel_manager.go
+++ b/flow/otel_metrics/otel_manager.go
@@ -17,46 +17,108 @@ import (
"github.com/PeerDB-io/peer-flow/peerdbenv"
)
+const (
+ SlotLagGaugeName string = "cdc_slot_lag"
+ OpenConnectionsGaugeName string = "open_connections"
+ OpenReplicationConnectionsGaugeName string = "open_replication_connections"
+ IntervalSinceLastNormalizeGaugeName string = "interval_since_last_normalize"
+ FetchedBytesCounterName string = "fetched_bytes"
+)
+
+type SlotMetricGauges struct {
+ SlotLagGauge metric.Float64Gauge
+ OpenConnectionsGauge metric.Int64Gauge
+ OpenReplicationConnectionsGauge metric.Int64Gauge
+ IntervalSinceLastNormalizeGauge metric.Float64Gauge
+ FetchedBytesCounter metric.Int64Counter
+}
+
+func BuildMetricName(baseName string) string {
+ return peerdbenv.GetPeerDBOtelMetricsNamespace() + baseName
+}
+
type OtelManager struct {
MetricsProvider *sdkmetric.MeterProvider
Meter metric.Meter
- Float64GaugesCache map[string]*Float64SyncGauge
- Int64GaugesCache map[string]*Int64SyncGauge
+ Float64GaugesCache map[string]metric.Float64Gauge
+ Int64GaugesCache map[string]metric.Int64Gauge
+ Int64CountersCache map[string]metric.Int64Counter
+}
+
+func NewOtelManager() (*OtelManager, error) {
+ metricsProvider, err := SetupPeerDBMetricsProvider("flow-worker")
+ if err != nil {
+ return nil, err
+ }
+
+ return &OtelManager{
+ MetricsProvider: metricsProvider,
+ Meter: metricsProvider.Meter("io.peerdb.flow-worker"),
+ Float64GaugesCache: make(map[string]metric.Float64Gauge),
+ Int64GaugesCache: make(map[string]metric.Int64Gauge),
+ Int64CountersCache: make(map[string]metric.Int64Counter),
+ }, nil
+}
+
+func (om *OtelManager) Close(ctx context.Context) error {
+ return om.MetricsProvider.Shutdown(ctx)
+}
+
+func getOrInitMetric[M any, O any](
+ cons func(metric.Meter, string, ...O) (M, error),
+ meter metric.Meter,
+ cache map[string]M,
+ name string,
+ opts ...O,
+) (M, error) {
+ gauge, ok := cache[name]
+ if !ok {
+ var err error
+ gauge, err = cons(meter, name, opts...)
+ if err != nil {
+ var none M
+ return none, err
+ }
+ cache[name] = gauge
+ }
+ return gauge, nil
+}
+
+func (om *OtelManager) GetOrInitInt64Gauge(name string, opts ...metric.Int64GaugeOption) (metric.Int64Gauge, error) {
+ return getOrInitMetric(metric.Meter.Int64Gauge, om.Meter, om.Int64GaugesCache, name, opts...)
+}
+
+func (om *OtelManager) GetOrInitFloat64Gauge(name string, opts ...metric.Float64GaugeOption) (metric.Float64Gauge, error) {
+ return getOrInitMetric(metric.Meter.Float64Gauge, om.Meter, om.Float64GaugesCache, name, opts...)
+}
+
+func (om *OtelManager) GetOrInitInt64Counter(name string, opts ...metric.Int64CounterOption) (metric.Int64Counter, error) {
+ return getOrInitMetric(metric.Meter.Int64Counter, om.Meter, om.Int64CountersCache, name, opts...)
}
// newOtelResource returns a resource describing this application.
func newOtelResource(otelServiceName string, attrs ...attribute.KeyValue) (*resource.Resource, error) {
- allAttrs := []attribute.KeyValue{
+ allAttrs := append([]attribute.KeyValue{
semconv.ServiceNameKey.String(otelServiceName),
- }
- allAttrs = append(allAttrs, attrs...)
- r, err := resource.Merge(
+ attribute.String(DeploymentUidKey, peerdbenv.PeerDBDeploymentUID()),
+ }, attrs...)
+ return resource.Merge(
resource.Default(),
resource.NewWithAttributes(
semconv.SchemaURL,
allAttrs...,
),
)
-
- return r, err
-}
-
-func setupHttpOtelMetricsExporter() (sdkmetric.Exporter, error) {
- return otlpmetrichttp.New(context.Background())
-}
-
-func setupGrpcOtelMetricsExporter() (sdkmetric.Exporter, error) {
- return otlpmetricgrpc.New(context.Background())
}
func temporalMetricsFilteringView() sdkmetric.View {
- exportListString := GetPeerDBOtelTemporalMetricsExportListEnv()
+ exportListString := peerdbenv.GetPeerDBOtelTemporalMetricsExportListEnv()
slog.Info("Found export list for temporal metrics", slog.String("exportList", exportListString))
// Special case for exporting all metrics
if exportListString == "__ALL__" {
return func(instrument sdkmetric.Instrument) (sdkmetric.Stream, bool) {
stream := sdkmetric.Stream{
- Name: GetPeerDBOtelMetricsNamespace() + "temporal." + instrument.Name,
+ Name: BuildMetricName("temporal." + instrument.Name),
Description: instrument.Description,
Unit: instrument.Unit,
}
@@ -68,7 +130,7 @@ func temporalMetricsFilteringView() sdkmetric.View {
if len(exportList) == 0 {
return func(instrument sdkmetric.Instrument) (sdkmetric.Stream, bool) {
return sdkmetric.Stream{
- Name: GetPeerDBOtelMetricsNamespace() + "temporal." + instrument.Name,
+ Name: BuildMetricName("temporal." + instrument.Name),
Description: instrument.Description,
Unit: instrument.Unit,
Aggregation: sdkmetric.AggregationDrop{},
@@ -84,7 +146,7 @@ func temporalMetricsFilteringView() sdkmetric.View {
}
return func(instrument sdkmetric.Instrument) (sdkmetric.Stream, bool) {
stream := sdkmetric.Stream{
- Name: GetPeerDBOtelMetricsNamespace() + "temporal." + instrument.Name,
+ Name: BuildMetricName("temporal." + instrument.Name),
Description: instrument.Description,
Unit: instrument.Unit,
}
@@ -95,16 +157,16 @@ func temporalMetricsFilteringView() sdkmetric.View {
}
}
-func setupExporter() (sdkmetric.Exporter, error) {
+func setupExporter(ctx context.Context) (sdkmetric.Exporter, error) {
otlpMetricProtocol := peerdbenv.GetEnvString("OTEL_EXPORTER_OTLP_PROTOCOL",
peerdbenv.GetEnvString("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", "http/protobuf"))
var metricExporter sdkmetric.Exporter
var err error
switch otlpMetricProtocol {
case "http/protobuf":
- metricExporter, err = setupHttpOtelMetricsExporter()
+ metricExporter, err = otlpmetrichttp.New(ctx)
case "grpc":
- metricExporter, err = setupGrpcOtelMetricsExporter()
+ metricExporter, err = otlpmetricgrpc.New(ctx)
default:
return nil, fmt.Errorf("unsupported otel metric protocol: %s", otlpMetricProtocol)
}
@@ -114,8 +176,8 @@ func setupExporter() (sdkmetric.Exporter, error) {
return metricExporter, err
}
-func setupMetricsProvider(otelResource *resource.Resource, views ...sdkmetric.View) (*sdkmetric.MeterProvider, error) {
- metricExporter, err := setupExporter()
+func setupMetricsProvider(ctx context.Context, otelResource *resource.Resource, views ...sdkmetric.View) (*sdkmetric.MeterProvider, error) {
+ metricExporter, err := setupExporter(ctx)
if err != nil {
return nil, err
}
@@ -133,13 +195,13 @@ func SetupPeerDBMetricsProvider(otelServiceName string) (*sdkmetric.MeterProvide
if err != nil {
return nil, fmt.Errorf("failed to create OpenTelemetry resource: %w", err)
}
- return setupMetricsProvider(otelResource)
+ return setupMetricsProvider(context.Background(), otelResource)
}
func SetupTemporalMetricsProvider(otelServiceName string) (*sdkmetric.MeterProvider, error) {
- otelResource, err := newOtelResource(otelServiceName, attribute.String(DeploymentUidKey, peerdbenv.PeerDBDeploymentUID()))
+ otelResource, err := newOtelResource(otelServiceName)
if err != nil {
return nil, fmt.Errorf("failed to create OpenTelemetry resource: %w", err)
}
- return setupMetricsProvider(otelResource, temporalMetricsFilteringView())
+ return setupMetricsProvider(context.Background(), otelResource, temporalMetricsFilteringView())
}
diff --git a/flow/otel_metrics/peerdb_gauges/gauges.go b/flow/otel_metrics/peerdb_gauges/gauges.go
deleted file mode 100644
index 767aac0945..0000000000
--- a/flow/otel_metrics/peerdb_gauges/gauges.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package peerdb_gauges
-
-import (
- "github.com/PeerDB-io/peer-flow/otel_metrics"
-)
-
-const (
- SlotLagGaugeName string = "cdc_slot_lag"
- OpenConnectionsGaugeName string = "open_connections"
- OpenReplicationConnectionsGaugeName string = "open_replication_connections"
- IntervalSinceLastNormalizeGaugeName string = "interval_since_last_normalize"
-)
-
-type SlotMetricGauges struct {
- SlotLagGauge *otel_metrics.Float64SyncGauge
- OpenConnectionsGauge *otel_metrics.Int64SyncGauge
- OpenReplicationConnectionsGauge *otel_metrics.Int64SyncGauge
- IntervalSinceLastNormalizeGauge *otel_metrics.Float64SyncGauge
-}
-
-func BuildGaugeName(baseGaugeName string) string {
- return otel_metrics.GetPeerDBOtelMetricsNamespace() + baseGaugeName
-}
diff --git a/flow/otel_metrics/sync_gauges.go b/flow/otel_metrics/sync_gauges.go
deleted file mode 100644
index d2ef4924c1..0000000000
--- a/flow/otel_metrics/sync_gauges.go
+++ /dev/null
@@ -1,125 +0,0 @@
-package otel_metrics
-
-import (
- "context"
- "fmt"
- "sync"
-
- "go.opentelemetry.io/otel/attribute"
- "go.opentelemetry.io/otel/metric"
-)
-
-type ObservationMapValue[V comparable] struct {
- Value V
-}
-
-// SyncGauge is a generic synchronous gauge that can be used to observe any type of value
-// Inspired from https://github.com/open-telemetry/opentelemetry-go/issues/3984#issuecomment-1743231837
-type SyncGauge[V comparable, O metric.Observable] struct {
- observableGauge O
- observations sync.Map
- name string
-}
-
-func (a *SyncGauge[V, O]) Callback(ctx context.Context, observeFunc func(value V, options ...metric.ObserveOption)) error {
- a.observations.Range(func(key, value interface{}) bool {
- attrs := key.(attribute.Set)
- val := value.(*ObservationMapValue[V])
- observeFunc(val.Value, metric.WithAttributeSet(attrs))
- // If the pointer is still same we can safely delete, else it means that the value was overwritten in parallel
- a.observations.CompareAndDelete(attrs, val)
- return true
- })
- return nil
-}
-
-func (a *SyncGauge[V, O]) Set(input V, attrs attribute.Set) {
- val := ObservationMapValue[V]{Value: input}
- a.observations.Store(attrs, &val)
-}
-
-type Int64SyncGauge struct {
- syncGauge *SyncGauge[int64, metric.Int64Observable]
-}
-
-func (a *Int64SyncGauge) Set(input int64, attrs attribute.Set) {
- if a == nil {
- return
- }
- a.syncGauge.Set(input, attrs)
-}
-
-func NewInt64SyncGauge(meter metric.Meter, gaugeName string, opts ...metric.Int64ObservableGaugeOption) (*Int64SyncGauge, error) {
- syncGauge := &SyncGauge[int64, metric.Int64Observable]{
- name: gaugeName,
- }
- observableGauge, err := meter.Int64ObservableGauge(gaugeName,
- append(opts, metric.WithInt64Callback(func(ctx context.Context, observer metric.Int64Observer) error {
- return syncGauge.Callback(ctx, func(value int64, options ...metric.ObserveOption) {
- observer.Observe(value, options...)
- })
- }))...)
- if err != nil {
- return nil, fmt.Errorf("failed to create Int64SyncGauge: %w", err)
- }
- syncGauge.observableGauge = observableGauge
- return &Int64SyncGauge{syncGauge: syncGauge}, nil
-}
-
-type Float64SyncGauge struct {
- syncGauge *SyncGauge[float64, metric.Float64Observable]
-}
-
-func (a *Float64SyncGauge) Set(input float64, attrs attribute.Set) {
- if a == nil {
- return
- }
- a.syncGauge.Set(input, attrs)
-}
-
-func NewFloat64SyncGauge(meter metric.Meter, gaugeName string, opts ...metric.Float64ObservableGaugeOption) (*Float64SyncGauge, error) {
- syncGauge := &SyncGauge[float64, metric.Float64Observable]{
- name: gaugeName,
- }
- observableGauge, err := meter.Float64ObservableGauge(gaugeName,
- append(opts, metric.WithFloat64Callback(func(ctx context.Context, observer metric.Float64Observer) error {
- return syncGauge.Callback(ctx, func(value float64, options ...metric.ObserveOption) {
- observer.Observe(value, options...)
- })
- }))...)
- if err != nil {
- return nil, fmt.Errorf("failed to create Float64SyncGauge: %w", err)
- }
- syncGauge.observableGauge = observableGauge
- return &Float64SyncGauge{syncGauge: syncGauge}, nil
-}
-
-func GetOrInitInt64SyncGauge(meter metric.Meter, cache map[string]*Int64SyncGauge, name string,
- opts ...metric.Int64ObservableGaugeOption,
-) (*Int64SyncGauge, error) {
- gauge, ok := cache[name]
- if !ok {
- var err error
- gauge, err = NewInt64SyncGauge(meter, name, opts...)
- if err != nil {
- return nil, err
- }
- cache[name] = gauge
- }
- return gauge, nil
-}
-
-func GetOrInitFloat64SyncGauge(meter metric.Meter, cache map[string]*Float64SyncGauge,
- name string, opts ...metric.Float64ObservableGaugeOption,
-) (*Float64SyncGauge, error) {
- gauge, ok := cache[name]
- if !ok {
- var err error
- gauge, err = NewFloat64SyncGauge(meter, name, opts...)
- if err != nil {
- return nil, err
- }
- cache[name] = gauge
- }
- return gauge, nil
-}
diff --git a/flow/peerdbenv/config.go b/flow/peerdbenv/config.go
index e033b87195..9aa9d2c5ed 100644
--- a/flow/peerdbenv/config.go
+++ b/flow/peerdbenv/config.go
@@ -166,3 +166,9 @@ func PeerDBRAPIRequestLoggingEnabled() bool {
}
return requestLoggingEnabled
}
+
+// PEERDB_MAINTENANCE_MODE_WAIT_ALERT_SECONDS tells how long to wait before alerting that peerdb has been stuck in maintenance mode
+// for too long
+func PeerDBMaintenanceModeWaitAlertSeconds() int {
+ return getEnvInt("PEERDB_MAINTENANCE_MODE_WAIT_ALERT_SECONDS", 600)
+}
diff --git a/flow/peerdbenv/dynamicconf.go b/flow/peerdbenv/dynamicconf.go
index 1e2f225906..98a47d8fdc 100644
--- a/flow/peerdbenv/dynamicconf.go
+++ b/flow/peerdbenv/dynamicconf.go
@@ -8,8 +8,10 @@ import (
"strconv"
"time"
+ "github.com/aws/smithy-go/ptr"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
+ "github.com/jackc/pgx/v5/pgxpool"
"golang.org/x/exp/constraints"
"github.com/PeerDB-io/peer-flow/generated/protos"
@@ -17,14 +19,6 @@ import (
)
var DynamicSettings = [...]*protos.DynamicSetting{
- {
- Name: "PEERDB_MAX_SYNCS_PER_CDC_FLOW",
- Description: "Experimental setting: changes number of syncs per workflow, affects frequency of replication slot disconnects",
- DefaultValue: "32",
- ValueType: protos.DynconfValueType_UINT,
- ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
- TargetForSetting: protos.DynconfTarget_ALL,
- },
{
Name: "PEERDB_CDC_CHANNEL_BUFFER_SIZE",
Description: "Advanced setting: changes buffer size of channel PeerDB uses while streaming rows read to destination in CDC",
@@ -68,17 +62,14 @@ var DynamicSettings = [...]*protos.DynamicSetting{
{
Name: "PEERDB_ENABLE_WAL_HEARTBEAT",
Description: "Enables WAL heartbeat to prevent replication slot lag from increasing during times of no activity",
- DefaultValue: "false",
+ DefaultValue: "true",
ValueType: protos.DynconfValueType_BOOL,
ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
TargetForSetting: protos.DynconfTarget_ALL,
},
{
- Name: "PEERDB_WAL_HEARTBEAT_QUERY",
- DefaultValue: `BEGIN;
-DROP AGGREGATE IF EXISTS PEERDB_EPHEMERAL_HEARTBEAT(float4);
-CREATE AGGREGATE PEERDB_EPHEMERAL_HEARTBEAT(float4) (SFUNC = float4pl, STYPE = float4);
-DROP AGGREGATE PEERDB_EPHEMERAL_HEARTBEAT(float4); END;`,
+ Name: "PEERDB_WAL_HEARTBEAT_QUERY",
+ DefaultValue: "SELECT pg_logical_emit_message(false,'peerdb_heartbeat','')",
ValueType: protos.DynconfValueType_STRING,
Description: "SQL to run during each WAL heartbeat",
ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
@@ -92,6 +83,13 @@ DROP AGGREGATE PEERDB_EPHEMERAL_HEARTBEAT(float4); END;`,
ApplyMode: protos.DynconfApplyMode_APPLY_MODE_AFTER_RESUME,
TargetForSetting: protos.DynconfTarget_ALL,
},
+ {
+ Name: "PEERDB_FULL_REFRESH_OVERWRITE_MODE",
+ Description: "Enables full refresh mode for query replication mirrors of overwrite type",
+ DefaultValue: "false",
+ ValueType: protos.DynconfValueType_BOOL,
+ ApplyMode: protos.DynconfApplyMode_APPLY_MODE_NEW_MIRROR,
+ },
{
Name: "PEERDB_NULLABLE",
Description: "Propagate nullability in schema",
@@ -116,6 +114,15 @@ DROP AGGREGATE PEERDB_EPHEMERAL_HEARTBEAT(float4); END;`,
ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
TargetForSetting: protos.DynconfTarget_CLICKHOUSE,
},
+ {
+ Name: "PEERDB_S3_PART_SIZE",
+ Description: "S3 upload part size in bytes, may need to increase for large batches. " +
+ "https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html",
+ DefaultValue: "0",
+ ValueType: protos.DynconfValueType_INT,
+ ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
+ TargetForSetting: protos.DynconfTarget_ALL,
+ },
{
Name: "PEERDB_QUEUE_FORCE_TOPIC_CREATION",
Description: "Force auto topic creation in mirrors, applies to Kafka and PubSub mirrors",
@@ -164,6 +171,30 @@ DROP AGGREGATE PEERDB_EPHEMERAL_HEARTBEAT(float4); END;`,
ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
TargetForSetting: protos.DynconfTarget_CLICKHOUSE,
},
+ {
+ Name: "PEERDB_CLICKHOUSE_MAX_INSERT_THREADS",
+ Description: "Configures max_insert_threads setting on clickhouse for inserting into destination table. Setting left unset when 0",
+ DefaultValue: "0",
+ ValueType: protos.DynconfValueType_UINT,
+ ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
+ TargetForSetting: protos.DynconfTarget_CLICKHOUSE,
+ },
+ {
+ Name: "PEERDB_CLICKHOUSE_PARALLEL_NORMALIZE",
+ Description: "Divide tables in batch into N insert selects. Helps distribute load to multiple nodes",
+ DefaultValue: "0",
+ ValueType: protos.DynconfValueType_INT,
+ ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
+ TargetForSetting: protos.DynconfTarget_CLICKHOUSE,
+ },
+ {
+ Name: "PEERDB_CLICKHOUSE_UNBOUNDED_NUMERIC_AS_STRING",
+ Description: "Map unbounded numerics in Postgres to String in ClickHouse to preserve precision and scale",
+ DefaultValue: "false",
+ ValueType: protos.DynconfValueType_BOOL,
+ ApplyMode: protos.DynconfApplyMode_APPLY_MODE_NEW_MIRROR,
+ TargetForSetting: protos.DynconfTarget_CLICKHOUSE,
+ },
{
Name: "PEERDB_INTERVAL_SINCE_LAST_NORMALIZE_THRESHOLD_MINUTES",
Description: "Duration in minutes since last normalize to start alerting, 0 disables all alerting entirely",
@@ -180,6 +211,14 @@ DROP AGGREGATE PEERDB_EPHEMERAL_HEARTBEAT(float4); END;`,
ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
TargetForSetting: protos.DynconfTarget_ALL,
},
+ {
+ Name: "PEERDB_MAINTENANCE_MODE_ENABLED",
+ Description: "Whether PeerDB is in maintenance mode, which disables any modifications to mirrors",
+ DefaultValue: "false",
+ ValueType: protos.DynconfValueType_BOOL,
+ ApplyMode: protos.DynconfApplyMode_APPLY_MODE_IMMEDIATE,
+ TargetForSetting: protos.DynconfTarget_ALL,
+ },
}
var DynamicIndex = func() map[string]int {
@@ -232,8 +271,8 @@ func dynamicConfSigned[T constraints.Signed](ctx context.Context, env map[string
return strconv.ParseInt(value, 10, 64)
})
if err != nil {
- shared.LoggerFromCtx(ctx).Error("Failed to parse as int64", slog.Any("error", err))
- return 0, fmt.Errorf("failed to parse as int64: %w", err)
+ shared.LoggerFromCtx(ctx).Error("Failed to parse as int64", slog.String("key", key), slog.Any("error", err))
+ return 0, fmt.Errorf("failed to parse %s as int64: %w", key, err)
}
return T(value), nil
@@ -244,8 +283,8 @@ func dynamicConfUnsigned[T constraints.Unsigned](ctx context.Context, env map[st
return strconv.ParseUint(value, 10, 64)
})
if err != nil {
- shared.LoggerFromCtx(ctx).Error("Failed to parse as uint64", slog.Any("error", err))
- return 0, fmt.Errorf("failed to parse as uint64: %w", err)
+ shared.LoggerFromCtx(ctx).Error("Failed to parse as uint64", slog.String("key", key), slog.Any("error", err))
+ return 0, fmt.Errorf("failed to parse %s as uint64: %w", key, err)
}
return T(value), nil
@@ -254,13 +293,27 @@ func dynamicConfUnsigned[T constraints.Unsigned](ctx context.Context, env map[st
func dynamicConfBool(ctx context.Context, env map[string]string, key string) (bool, error) {
value, err := dynLookupConvert(ctx, env, key, strconv.ParseBool)
if err != nil {
- shared.LoggerFromCtx(ctx).Error("Failed to parse bool", slog.Any("error", err))
- return false, fmt.Errorf("failed to parse bool: %w", err)
+ shared.LoggerFromCtx(ctx).Error("Failed to parse bool", slog.String("key", key), slog.Any("error", err))
+ return false, fmt.Errorf("failed to parse %s as bool: %w", key, err)
}
return value, nil
}
+func UpdateDynamicSetting(ctx context.Context, pool *pgxpool.Pool, name string, value *string) error {
+ if pool == nil {
+ var err error
+ pool, err = GetCatalogConnectionPoolFromEnv(ctx)
+ if err != nil {
+ shared.LoggerFromCtx(ctx).Error("Failed to get catalog connection pool for dynamic setting update", slog.Any("error", err))
+ return fmt.Errorf("failed to get catalog connection pool: %w", err)
+ }
+ }
+ _, err := pool.Exec(ctx, `insert into dynamic_settings (config_name, config_value) values ($1, $2)
+ on conflict (config_name) do update set config_value = $2`, name, value)
+ return err
+}
+
// PEERDB_SLOT_LAG_MB_ALERT_THRESHOLD, 0 disables slot lag alerting entirely
func PeerDBSlotLagMBAlertThreshold(ctx context.Context, env map[string]string) (uint32, error) {
return dynamicConfUnsigned[uint32](ctx, env, "PEERDB_SLOT_LAG_MB_ALERT_THRESHOLD")
@@ -324,6 +377,10 @@ func PeerDBEnableParallelSyncNormalize(ctx context.Context, env map[string]strin
return dynamicConfBool(ctx, env, "PEERDB_ENABLE_PARALLEL_SYNC_NORMALIZE")
}
+func PeerDBFullRefreshOverwriteMode(ctx context.Context, env map[string]string) (bool, error) {
+ return dynamicConfBool(ctx, env, "PEERDB_FULL_REFRESH_OVERWRITE_MODE")
+}
+
func PeerDBNullable(ctx context.Context, env map[string]string) (bool, error) {
return dynamicConfBool(ctx, env, "PEERDB_NULLABLE")
}
@@ -332,6 +389,18 @@ func PeerDBEnableClickHousePrimaryUpdate(ctx context.Context, env map[string]str
return dynamicConfBool(ctx, env, "PEERDB_CLICKHOUSE_ENABLE_PRIMARY_UPDATE")
}
+func PeerDBClickHouseMaxInsertThreads(ctx context.Context, env map[string]string) (int64, error) {
+ return dynamicConfSigned[int64](ctx, env, "PEERDB_CLICKHOUSE_MAX_INSERT_THREADS")
+}
+
+func PeerDBClickHouseParallelNormalize(ctx context.Context, env map[string]string) (int, error) {
+ return dynamicConfSigned[int](ctx, env, "PEERDB_CLICKHOUSE_PARALLEL_NORMALIZE")
+}
+
+func PeerDBEnableClickHouseNumericAsString(ctx context.Context, env map[string]string) (bool, error) {
+ return dynamicConfBool(ctx, env, "PEERDB_CLICKHOUSE_UNBOUNDED_NUMERIC_AS_STRING")
+}
+
func PeerDBSnowflakeMergeParallelism(ctx context.Context, env map[string]string) (int64, error) {
return dynamicConfSigned[int64](ctx, env, "PEERDB_SNOWFLAKE_MERGE_PARALLELISM")
}
@@ -340,6 +409,10 @@ func PeerDBClickHouseAWSS3BucketName(ctx context.Context, env map[string]string)
return dynLookup(ctx, env, "PEERDB_CLICKHOUSE_AWS_S3_BUCKET_NAME")
}
+func PeerDBS3PartSize(ctx context.Context, env map[string]string) (int64, error) {
+ return dynamicConfSigned[int64](ctx, env, "PEERDB_S3_PART_SIZE")
+}
+
// Kafka has topic auto create as an option, auto.create.topics.enable
// But non-dedicated cluster maybe can't set config, may want peerdb to create topic. Similar for PubSub
func PeerDBQueueForceTopicCreation(ctx context.Context, env map[string]string) (bool, error) {
@@ -354,3 +427,11 @@ func PeerDBIntervalSinceLastNormalizeThresholdMinutes(ctx context.Context, env m
func PeerDBApplicationNamePerMirrorName(ctx context.Context, env map[string]string) (bool, error) {
return dynamicConfBool(ctx, env, "PEERDB_APPLICATION_NAME_PER_MIRROR_NAME")
}
+
+func PeerDBMaintenanceModeEnabled(ctx context.Context, env map[string]string) (bool, error) {
+ return dynamicConfBool(ctx, env, "PEERDB_MAINTENANCE_MODE_ENABLED")
+}
+
+func UpdatePeerDBMaintenanceModeEnabled(ctx context.Context, pool *pgxpool.Pool, enabled bool) error {
+ return UpdateDynamicSetting(ctx, pool, "PEERDB_MAINTENANCE_MODE_ENABLED", ptr.String(strconv.FormatBool(enabled)))
+}
diff --git a/flow/peerdbenv/otel.go b/flow/peerdbenv/otel.go
new file mode 100644
index 0000000000..d7f3cb68a6
--- /dev/null
+++ b/flow/peerdbenv/otel.go
@@ -0,0 +1,9 @@
+package peerdbenv
+
+func GetPeerDBOtelMetricsNamespace() string {
+ return GetEnvString("PEERDB_OTEL_METRICS_NAMESPACE", "")
+}
+
+func GetPeerDBOtelTemporalMetricsExportListEnv() string {
+ return GetEnvString("PEERDB_OTEL_TEMPORAL_METRICS_EXPORT_LIST", "")
+}
diff --git a/flow/shared/constants.go b/flow/shared/constants.go
index 2dc5a8a64e..955ecfc4b5 100644
--- a/flow/shared/constants.go
+++ b/flow/shared/constants.go
@@ -11,8 +11,9 @@ type (
const (
// Task Queues
- PeerFlowTaskQueue TaskQueueID = "peer-flow-task-queue"
- SnapshotFlowTaskQueue TaskQueueID = "snapshot-flow-task-queue"
+ PeerFlowTaskQueue TaskQueueID = "peer-flow-task-queue"
+ SnapshotFlowTaskQueue TaskQueueID = "snapshot-flow-task-queue"
+ MaintenanceFlowTaskQueue TaskQueueID = "maintenance-flow-task-queue"
// Queries
CDCFlowStateQuery = "q-cdc-flow-state"
diff --git a/flow/shared/postgres.go b/flow/shared/postgres.go
index be3cf7d07d..121fb73bf4 100644
--- a/flow/shared/postgres.go
+++ b/flow/shared/postgres.go
@@ -58,17 +58,17 @@ func GetCustomDataTypes(ctx context.Context, conn *pgx.Conn) (map[uint32]string,
AND n.nspname NOT IN ('pg_catalog', 'information_schema');
`)
if err != nil {
- return nil, fmt.Errorf("failed to get custom types: %w", err)
+ return nil, fmt.Errorf("failed to get customTypeMapping: %w", err)
}
customTypeMap := map[uint32]string{}
- for rows.Next() {
- var typeID pgtype.Uint32
- var typeName pgtype.Text
- if err := rows.Scan(&typeID, &typeName); err != nil {
- return nil, fmt.Errorf("failed to scan row: %w", err)
- }
+ var typeID pgtype.Uint32
+ var typeName pgtype.Text
+ if _, err := pgx.ForEachRow(rows, []any{&typeID, &typeName}, func() error {
customTypeMap[typeID.Uint32] = typeName.String
+ return nil
+ }); err != nil {
+ return nil, fmt.Errorf("failed to scan into customTypeMapping: %w", err)
}
return customTypeMap, nil
}
diff --git a/flow/shared/random.go b/flow/shared/random.go
index 7ef3c8e5dc..84830f3762 100644
--- a/flow/shared/random.go
+++ b/flow/shared/random.go
@@ -2,32 +2,8 @@ package shared
import (
"crypto/rand"
- "encoding/binary"
- "errors"
)
-// RandomInt64 returns a random 64 bit integer.
-func RandomInt64() (int64, error) {
- b := make([]byte, 8)
- _, err := rand.Read(b)
- if err != nil {
- return 0, errors.New("could not generate random int64: " + err.Error())
- }
- // Convert bytes to int64
- return int64(binary.LittleEndian.Uint64(b)), nil
-}
-
-// RandomUInt64 returns a random 64 bit unsigned integer.
-func RandomUInt64() (uint64, error) {
- b := make([]byte, 8)
- _, err := rand.Read(b)
- if err != nil {
- return 0, errors.New("could not generate random uint64: " + err.Error())
- }
- // Convert bytes to uint64
- return binary.LittleEndian.Uint64(b), nil
-}
-
func RandomString(n int) string {
const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
bytes := make([]byte, n)
diff --git a/flow/shared/telemetry/event_types.go b/flow/shared/telemetry/event_types.go
index 0d87ba3540..a68fab869f 100644
--- a/flow/shared/telemetry/event_types.go
+++ b/flow/shared/telemetry/event_types.go
@@ -3,7 +3,11 @@ package telemetry
type EventType string
const (
- CreatePeer EventType = "CreatePeer"
- CreateMirror EventType = "CreateMirror"
- Other EventType = "Other"
+ CreatePeer EventType = "CreatePeer"
+ CreateMirror EventType = "CreateMirror"
+ StartMaintenance EventType = "StartMaintenance"
+ EndMaintenance EventType = "EndMaintenance"
+ MaintenanceWait EventType = "MaintenanceWait"
+
+ Other EventType = "Other"
)
diff --git a/flow/shared/workflow.go b/flow/shared/workflow.go
new file mode 100644
index 0000000000..c9cafc37e2
--- /dev/null
+++ b/flow/shared/workflow.go
@@ -0,0 +1,27 @@
+package shared
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+
+ "go.temporal.io/sdk/client"
+
+ "github.com/PeerDB-io/peer-flow/generated/protos"
+)
+
+func GetWorkflowStatus(ctx context.Context, temporalClient client.Client, workflowID string) (protos.FlowStatus, error) {
+ res, err := temporalClient.QueryWorkflow(ctx, workflowID, "", FlowStatusQuery)
+ if err != nil {
+ slog.Error("failed to query status in workflow with ID "+workflowID, slog.Any("error", err))
+ return protos.FlowStatus_STATUS_UNKNOWN,
+ fmt.Errorf("failed to query status in workflow with ID %s: %w", workflowID, err)
+ }
+ var state protos.FlowStatus
+ if err := res.Get(&state); err != nil {
+ slog.Error("failed to get status in workflow with ID "+workflowID, slog.Any("error", err))
+ return protos.FlowStatus_STATUS_UNKNOWN,
+ fmt.Errorf("failed to get status in workflow with ID %s: %w", workflowID, err)
+ }
+ return state, nil
+}
diff --git a/flow/tags/tags.go b/flow/tags/tags.go
new file mode 100644
index 0000000000..8adc9a437b
--- /dev/null
+++ b/flow/tags/tags.go
@@ -0,0 +1,24 @@
+package tags
+
+import (
+ "context"
+ "log/slog"
+
+ "github.com/jackc/pgx/v5/pgxpool"
+)
+
+func GetTags(ctx context.Context, catalogPool *pgxpool.Pool, flowName string) (map[string]string, error) {
+ var tags map[string]string
+
+ err := catalogPool.QueryRow(ctx, "SELECT tags FROM flows WHERE name = $1", flowName).Scan(&tags)
+ if err != nil {
+ slog.Error("error getting flow tags", slog.Any("error", err))
+ return nil, err
+ }
+
+ if tags == nil {
+ tags = make(map[string]string)
+ }
+
+ return tags, nil
+}
diff --git a/flow/workflows/activities.go b/flow/workflows/activities.go
index 0b23d10dd1..5fe699419c 100644
--- a/flow/workflows/activities.go
+++ b/flow/workflows/activities.go
@@ -3,6 +3,7 @@ package peerflow
import "github.com/PeerDB-io/peer-flow/activities"
var (
- flowable *activities.FlowableActivity
- snapshot *activities.SnapshotActivity
+ flowable *activities.FlowableActivity
+ snapshot *activities.SnapshotActivity
+ maintenance *activities.MaintenanceActivity
)
diff --git a/flow/workflows/cdc_flow.go b/flow/workflows/cdc_flow.go
index 72e37b01fd..0c97af9b7d 100644
--- a/flow/workflows/cdc_flow.go
+++ b/flow/workflows/cdc_flow.go
@@ -480,13 +480,15 @@ func CDCFlowWorkflow(
}
}
- state.CurrentFlowStatus = protos.FlowStatus_STATUS_RUNNING
logger.Info("executed setup flow and snapshot flow")
-
// if initial_copy_only is opted for, we end the flow here.
if cfg.InitialSnapshotOnly {
+ logger.Info("initial snapshot only, ending flow")
+ state.CurrentFlowStatus = protos.FlowStatus_STATUS_COMPLETED
return state, nil
}
+
+ state.CurrentFlowStatus = protos.FlowStatus_STATUS_RUNNING
}
syncFlowID := GetChildWorkflowID("sync-flow", cfg.FlowJobName, originalRunID)
diff --git a/flow/workflows/drop_flow.go b/flow/workflows/drop_flow.go
index 51bf0091a1..93086157d8 100644
--- a/flow/workflows/drop_flow.go
+++ b/flow/workflows/drop_flow.go
@@ -92,6 +92,15 @@ func DropFlowWorkflow(ctx workflow.Context, input *protos.DropFlowInput) error {
}
}
+ if input.FlowConnectionConfigs != nil {
+ err := executeCDCDropActivities(ctx, input)
+ if err != nil {
+ workflow.GetLogger(ctx).Error("failed to drop CDC flow", slog.Any("error", err))
+ return err
+ }
+ workflow.GetLogger(ctx).Info("CDC flow dropped successfully")
+ }
+
removeFlowEntriesCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
StartToCloseTimeout: 1 * time.Minute,
})
@@ -103,14 +112,5 @@ func DropFlowWorkflow(ctx workflow.Context, input *protos.DropFlowInput) error {
return err
}
- if input.FlowConnectionConfigs != nil {
- err := executeCDCDropActivities(ctx, input)
- if err != nil {
- workflow.GetLogger(ctx).Error("failed to drop CDC flow", slog.Any("error", err))
- return err
- }
- workflow.GetLogger(ctx).Info("CDC flow dropped successfully")
- }
-
return nil
}
diff --git a/flow/workflows/local_activities.go b/flow/workflows/local_activities.go
index d163352ca2..7a3e80f240 100644
--- a/flow/workflows/local_activities.go
+++ b/flow/workflows/local_activities.go
@@ -29,6 +29,20 @@ func getParallelSyncNormalize(wCtx workflow.Context, logger log.Logger, env map[
return parallel
}
+func getQRepOverwriteFullRefreshMode(wCtx workflow.Context, logger log.Logger, env map[string]string) bool {
+ checkCtx := workflow.WithLocalActivityOptions(wCtx, workflow.LocalActivityOptions{
+ StartToCloseTimeout: time.Minute,
+ })
+
+ getFullRefreshFuture := workflow.ExecuteLocalActivity(checkCtx, peerdbenv.PeerDBFullRefreshOverwriteMode, env)
+ var fullRefreshEnabled bool
+ if err := getFullRefreshFuture.Get(checkCtx, &fullRefreshEnabled); err != nil {
+ logger.Warn("Failed to check if full refresh mode is enabled", slog.Any("error", err))
+ return false
+ }
+ return fullRefreshEnabled
+}
+
func localPeerType(ctx context.Context, name string) (protos.DBType, error) {
pool, err := peerdbenv.GetCatalogConnectionPoolFromEnv(ctx)
if err != nil {
diff --git a/flow/workflows/maintenance_flow.go b/flow/workflows/maintenance_flow.go
new file mode 100644
index 0000000000..c48750a807
--- /dev/null
+++ b/flow/workflows/maintenance_flow.go
@@ -0,0 +1,305 @@
+package peerflow
+
+import (
+ "context"
+ "log/slog"
+ "time"
+
+ tEnums "go.temporal.io/api/enums/v1"
+ "go.temporal.io/sdk/client"
+ "go.temporal.io/sdk/log"
+ "go.temporal.io/sdk/workflow"
+
+ "github.com/PeerDB-io/peer-flow/generated/protos"
+ "github.com/PeerDB-io/peer-flow/peerdbenv"
+ "github.com/PeerDB-io/peer-flow/shared"
+)
+
+func getMaintenanceWorkflowOptions(workflowIDPrefix string, taskQueueId shared.TaskQueueID) client.StartWorkflowOptions {
+ maintenanceWorkflowOptions := client.StartWorkflowOptions{
+ WorkflowIDReusePolicy: tEnums.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE,
+ WorkflowIDConflictPolicy: tEnums.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING,
+ TaskQueue: peerdbenv.PeerFlowTaskQueueName(taskQueueId),
+ ID: workflowIDPrefix,
+ }
+ if deploymentUid := peerdbenv.PeerDBDeploymentUID(); deploymentUid != "" {
+ maintenanceWorkflowOptions.ID += "-" + deploymentUid
+ }
+ return maintenanceWorkflowOptions
+}
+
+// RunStartMaintenanceWorkflow is a helper function to start the StartMaintenanceWorkflow with sane defaults
+func RunStartMaintenanceWorkflow(
+ ctx context.Context,
+ temporalClient client.Client,
+ input *protos.StartMaintenanceFlowInput,
+ taskQueueId shared.TaskQueueID,
+) (client.WorkflowRun, error) {
+ workflowOptions := getMaintenanceWorkflowOptions("start-maintenance", taskQueueId)
+ workflowRun, err := temporalClient.ExecuteWorkflow(ctx, workflowOptions, StartMaintenanceWorkflow, input)
+ if err != nil {
+ return nil, err
+ }
+ return workflowRun, nil
+}
+
+// RunEndMaintenanceWorkflow is a helper function to start the EndMaintenanceWorkflow with sane defaults
+func RunEndMaintenanceWorkflow(
+ ctx context.Context,
+ temporalClient client.Client,
+ input *protos.EndMaintenanceFlowInput,
+ taskQueueId shared.TaskQueueID,
+) (client.WorkflowRun, error) {
+ workflowOptions := getMaintenanceWorkflowOptions("end-maintenance", taskQueueId)
+ workflowRun, err := temporalClient.ExecuteWorkflow(ctx, workflowOptions, EndMaintenanceWorkflow, &protos.EndMaintenanceFlowInput{})
+ if err != nil {
+ return nil, err
+ }
+ return workflowRun, nil
+}
+
+func StartMaintenanceWorkflow(ctx workflow.Context, input *protos.StartMaintenanceFlowInput) (*protos.StartMaintenanceFlowOutput, error) {
+ logger := workflow.GetLogger(ctx)
+ logger.Info("Starting StartMaintenance workflow", "input", input)
+ defer runBackgroundAlerter(ctx)()
+
+ maintenanceFlowOutput, err := startMaintenance(ctx, logger)
+ if err != nil {
+ slog.Error("Error in StartMaintenance workflow", "error", err)
+ return nil, err
+ }
+ return maintenanceFlowOutput, nil
+}
+
+func startMaintenance(ctx workflow.Context, logger log.Logger) (*protos.StartMaintenanceFlowOutput, error) {
+ ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 24 * time.Hour,
+ })
+
+ snapshotWaitCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 24 * time.Hour,
+ HeartbeatTimeout: 1 * time.Minute,
+ })
+ waitSnapshotsFuture := workflow.ExecuteActivity(snapshotWaitCtx,
+ maintenance.WaitForRunningSnapshots,
+ )
+ err := waitSnapshotsFuture.Get(snapshotWaitCtx, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ enableCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 5 * time.Minute,
+ })
+ enableMaintenanceFuture := workflow.ExecuteActivity(enableCtx, maintenance.EnableMaintenanceMode)
+
+ if err := enableMaintenanceFuture.Get(enableCtx, nil); err != nil {
+ return nil, err
+ }
+
+ logger.Info("Waiting for all snapshot mirrors to finish snapshotting")
+ waitSnapshotsPostEnableFuture := workflow.ExecuteActivity(snapshotWaitCtx,
+ maintenance.WaitForRunningSnapshots,
+ )
+
+ if err := waitSnapshotsPostEnableFuture.Get(snapshotWaitCtx, nil); err != nil {
+ return nil, err
+ }
+
+ mirrorsList, err := getAllMirrors(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ runningMirrors, err := pauseAndGetRunningMirrors(ctx, mirrorsList, logger)
+ if err != nil {
+ return nil, err
+ }
+
+ backupCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 2 * time.Minute,
+ })
+ future := workflow.ExecuteActivity(backupCtx, maintenance.BackupAllPreviouslyRunningFlows, runningMirrors)
+
+ if err := future.Get(backupCtx, nil); err != nil {
+ return nil, err
+ }
+ version, err := GetPeerDBVersion(ctx)
+ if err != nil {
+ return nil, err
+ }
+ logger.Info("StartMaintenance workflow completed", "version", version)
+ return &protos.StartMaintenanceFlowOutput{
+ Version: version,
+ }, nil
+}
+
+func pauseAndGetRunningMirrors(
+ ctx workflow.Context,
+ mirrorsList *protos.MaintenanceMirrors,
+ logger log.Logger,
+) (*protos.MaintenanceMirrors, error) {
+ ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 24 * time.Hour,
+ HeartbeatTimeout: 1 * time.Minute,
+ })
+ selector := workflow.NewSelector(ctx)
+ runningMirrors := make([]bool, len(mirrorsList.Mirrors))
+ for i, mirror := range mirrorsList.Mirrors {
+ f := workflow.ExecuteActivity(
+ ctx,
+ maintenance.PauseMirrorIfRunning,
+ mirror,
+ )
+
+ selector.AddFuture(f, func(f workflow.Future) {
+ var wasRunning bool
+ err := f.Get(ctx, &wasRunning)
+ if err != nil {
+ logger.Error("Error checking and pausing mirror", "mirror", mirror, "error", err)
+ } else {
+ logger.Info("Finished check and pause for mirror", "mirror", mirror, "wasRunning", wasRunning)
+ runningMirrors[i] = wasRunning
+ }
+ })
+ }
+ onlyRunningMirrors := make([]*protos.MaintenanceMirror, 0, len(mirrorsList.Mirrors))
+ for range mirrorsList.Mirrors {
+ selector.Select(ctx)
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ }
+ for i, mirror := range mirrorsList.Mirrors {
+ if runningMirrors[i] {
+ onlyRunningMirrors = append(onlyRunningMirrors, mirror)
+ }
+ }
+ return &protos.MaintenanceMirrors{
+ Mirrors: onlyRunningMirrors,
+ }, nil
+}
+
+func getAllMirrors(ctx workflow.Context) (*protos.MaintenanceMirrors, error) {
+ ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 2 * time.Minute,
+ })
+ getMirrorsFuture := workflow.ExecuteActivity(ctx, maintenance.GetAllMirrors)
+ var mirrorsList protos.MaintenanceMirrors
+ err := getMirrorsFuture.Get(ctx, &mirrorsList)
+ return &mirrorsList, err
+}
+
+func EndMaintenanceWorkflow(ctx workflow.Context, input *protos.EndMaintenanceFlowInput) (*protos.EndMaintenanceFlowOutput, error) {
+ logger := workflow.GetLogger(ctx)
+ logger.Info("Starting EndMaintenance workflow", "input", input)
+ defer runBackgroundAlerter(ctx)()
+
+ flowOutput, err := endMaintenance(ctx, logger)
+ if err != nil {
+ slog.Error("Error in EndMaintenance workflow", "error", err)
+ return nil, err
+ }
+ return flowOutput, nil
+}
+
+func endMaintenance(ctx workflow.Context, logger log.Logger) (*protos.EndMaintenanceFlowOutput, error) {
+ ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 24 * time.Hour,
+ HeartbeatTimeout: 1 * time.Minute,
+ })
+
+ mirrorsList, err := resumeBackedUpMirrors(ctx, logger)
+ if err != nil {
+ return nil, err
+ }
+
+ clearBackupsFuture := workflow.ExecuteActivity(ctx, maintenance.CleanBackedUpFlows)
+ if err := clearBackupsFuture.Get(ctx, nil); err != nil {
+ return nil, err
+ }
+
+ logger.Info("Resumed backed up mirrors", "mirrors", mirrorsList)
+
+ disableCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 5 * time.Minute,
+ })
+
+ future := workflow.ExecuteActivity(disableCtx, maintenance.DisableMaintenanceMode)
+ if err := future.Get(disableCtx, nil); err != nil {
+ return nil, err
+ }
+ logger.Info("Disabled maintenance mode")
+ version, err := GetPeerDBVersion(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ logger.Info("EndMaintenance workflow completed", "version", version)
+ return &protos.EndMaintenanceFlowOutput{
+ Version: version,
+ }, nil
+}
+
+func resumeBackedUpMirrors(ctx workflow.Context, logger log.Logger) (*protos.MaintenanceMirrors, error) {
+ future := workflow.ExecuteActivity(ctx, maintenance.GetBackedUpFlows)
+ ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{
+ StartToCloseTimeout: 5 * time.Minute,
+ })
+ var mirrorsList *protos.MaintenanceMirrors
+ err := future.Get(ctx, &mirrorsList)
+ if err != nil {
+ return nil, err
+ }
+
+ selector := workflow.NewSelector(ctx)
+ for _, mirror := range mirrorsList.Mirrors {
+ activityInput := mirror
+ f := workflow.ExecuteActivity(
+ ctx,
+ maintenance.ResumeMirror,
+ activityInput,
+ )
+
+ selector.AddFuture(f, func(f workflow.Future) {
+ err := f.Get(ctx, nil)
+ if err != nil {
+ logger.Error("Error resuming mirror", "mirror", mirror, "error", err)
+ } else {
+ logger.Info("Finished resuming mirror", "mirror", mirror)
+ }
+ })
+ }
+
+ for range mirrorsList.Mirrors {
+ selector.Select(ctx)
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ }
+ return mirrorsList, nil
+}
+
+// runBackgroundAlerter Alerts every few minutes regarding currently running maintenance workflows
+func runBackgroundAlerter(ctx workflow.Context) workflow.CancelFunc {
+ activityCtx, cancelActivity := workflow.WithCancel(ctx)
+ alerterCtx := workflow.WithActivityOptions(activityCtx, workflow.ActivityOptions{
+ StartToCloseTimeout: 24 * time.Hour,
+ HeartbeatTimeout: 1 * time.Minute,
+ })
+ workflow.ExecuteActivity(alerterCtx, maintenance.BackgroundAlerter)
+ return cancelActivity
+}
+
+func GetPeerDBVersion(wCtx workflow.Context) (string, error) {
+ activityCtx := workflow.WithLocalActivityOptions(wCtx, workflow.LocalActivityOptions{
+ StartToCloseTimeout: time.Minute,
+ })
+ getVersionActivity := func(ctx context.Context) (string, error) {
+ return peerdbenv.PeerDBVersionShaShort(), nil
+ }
+ var version string
+ future := workflow.ExecuteLocalActivity(activityCtx, getVersionActivity)
+ err := future.Get(activityCtx, &version)
+ return version, err
+}
diff --git a/flow/workflows/qrep_flow.go b/flow/workflows/qrep_flow.go
index c7348eefa9..f862b4f3d6 100644
--- a/flow/workflows/qrep_flow.go
+++ b/flow/workflows/qrep_flow.go
@@ -32,13 +32,15 @@ type QRepPartitionFlowExecution struct {
runUUID string
}
+var InitialLastPartition = &protos.QRepPartition{
+ PartitionId: "not-applicable-partition",
+ Range: nil,
+}
+
// returns a new empty QRepFlowState
func newQRepFlowState() *protos.QRepFlowState {
return &protos.QRepFlowState{
- LastPartition: &protos.QRepPartition{
- PartitionId: "not-applicable-partition",
- Range: nil,
- },
+ LastPartition: InitialLastPartition,
NumPartitionsProcessed: 0,
NeedsResync: true,
CurrentFlowStatus: protos.FlowStatus_STATUS_RUNNING,
@@ -461,8 +463,10 @@ func QRepWaitForNewRowsWorkflow(ctx workflow.Context, config *protos.QRepConfig,
return fmt.Errorf("error checking for new rows: %w", err)
}
+ optedForOverwrite := config.WriteMode.WriteType == protos.QRepWriteType_QREP_WRITE_MODE_OVERWRITE
+ fullRefresh := optedForOverwrite && getQRepOverwriteFullRefreshMode(ctx, logger, config.Env)
// If no new rows are found, continue as new
- if !hasNewRows {
+ if !hasNewRows || fullRefresh {
waitBetweenBatches := 5 * time.Second
if config.WaitBetweenBatchesSeconds > 0 {
waitBetweenBatches = time.Duration(config.WaitBetweenBatchesSeconds) * time.Second
@@ -472,6 +476,9 @@ func QRepWaitForNewRowsWorkflow(ctx workflow.Context, config *protos.QRepConfig,
return sleepErr
}
+ if fullRefresh {
+ return nil
+ }
logger.Info("QRepWaitForNewRowsWorkflow: continuing the loop")
return workflow.NewContinueAsNewError(ctx, QRepWaitForNewRowsWorkflow, config, lastPartition)
}
@@ -545,8 +552,16 @@ func QRepFlowWorkflow(
return state, err
}
- if !config.InitialCopyOnly && state.LastPartition != nil {
- if err := q.waitForNewRows(ctx, signalChan, state.LastPartition); err != nil {
+ fullRefresh := false
+ lastPartition := state.LastPartition
+ if config.WriteMode.WriteType == protos.QRepWriteType_QREP_WRITE_MODE_OVERWRITE {
+ if fullRefresh = getQRepOverwriteFullRefreshMode(ctx, q.logger, config.Env); fullRefresh {
+ lastPartition = InitialLastPartition
+ }
+ }
+
+ if !config.InitialCopyOnly && lastPartition != nil {
+ if err := q.waitForNewRows(ctx, signalChan, lastPartition); err != nil {
return state, err
}
}
@@ -580,7 +595,7 @@ func QRepFlowWorkflow(
q.logger.Info(fmt.Sprintf("%d partitions processed", len(partitions.Partitions)))
state.NumPartitionsProcessed += uint64(len(partitions.Partitions))
- if len(partitions.Partitions) > 0 {
+ if len(partitions.Partitions) > 0 && !fullRefresh {
state.LastPartition = partitions.Partitions[len(partitions.Partitions)-1]
}
}
diff --git a/flow/workflows/register.go b/flow/workflows/register.go
index 35adf135bf..2c4b32ba3c 100644
--- a/flow/workflows/register.go
+++ b/flow/workflows/register.go
@@ -18,4 +18,7 @@ func RegisterFlowWorkerWorkflows(w worker.WorkflowRegistry) {
w.RegisterWorkflow(GlobalScheduleManagerWorkflow)
w.RegisterWorkflow(HeartbeatFlowWorkflow)
w.RegisterWorkflow(RecordSlotSizeWorkflow)
+
+ w.RegisterWorkflow(StartMaintenanceWorkflow)
+ w.RegisterWorkflow(EndMaintenanceWorkflow)
}
diff --git a/flow/workflows/snapshot_flow.go b/flow/workflows/snapshot_flow.go
index c8b6a3fd29..1db3b6d60b 100644
--- a/flow/workflows/snapshot_flow.go
+++ b/flow/workflows/snapshot_flow.go
@@ -166,7 +166,7 @@ func (s *SnapshotFlowExecution) cloneTable(
numWorkers = s.config.SnapshotMaxParallelWorkers
}
- numRowsPerPartition := uint32(500000)
+ numRowsPerPartition := uint32(250000)
if s.config.SnapshotNumRowsPerPartition > 0 {
numRowsPerPartition = s.config.SnapshotNumRowsPerPartition
}
@@ -208,6 +208,7 @@ func (s *SnapshotFlowExecution) cloneTable(
WriteMode: snapshotWriteMode,
System: s.config.System,
Script: s.config.Script,
+ Env: s.config.Env,
ParentMirrorName: flowName,
}
@@ -274,6 +275,13 @@ func (s *SnapshotFlowExecution) cloneTablesWithSlot(
if err != nil {
return fmt.Errorf("failed to setup replication: %w", err)
}
+ defer func() {
+ dCtx, cancel := workflow.NewDisconnectedContext(sessionCtx)
+ defer cancel()
+ if err := s.closeSlotKeepAlive(dCtx); err != nil {
+ s.logger.Error("failed to close slot keep alive", slog.Any("error", err))
+ }
+ }()
s.logger.Info(fmt.Sprintf("cloning %d tables in parallel", numTablesInParallel))
if err := s.cloneTables(ctx,
@@ -283,13 +291,10 @@ func (s *SnapshotFlowExecution) cloneTablesWithSlot(
slotInfo.SupportsTidScans,
numTablesInParallel,
); err != nil {
+ s.logger.Error("failed to clone tables", slog.Any("error", err))
return fmt.Errorf("failed to clone tables: %w", err)
}
- if err := s.closeSlotKeepAlive(sessionCtx); err != nil {
- return fmt.Errorf("failed to close slot keep alive: %w", err)
- }
-
return nil
}
diff --git a/images/in-clickpipes.png b/images/in-clickpipes.png
new file mode 100644
index 0000000000..18d4c709d3
Binary files /dev/null and b/images/in-clickpipes.png differ
diff --git a/nexus/Cargo.lock b/nexus/Cargo.lock
index cc3650b6f8..9cd70344bb 100644
--- a/nexus/Cargo.lock
+++ b/nexus/Cargo.lock
@@ -785,9 +785,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
-version = "1.8.0"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da"
+checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b"
[[package]]
name = "bytes-utils"
@@ -801,9 +801,9 @@ dependencies = [
[[package]]
name = "cargo-deb"
-version = "2.8.0"
+version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74d0500c935971265437386796faad57064d17bf2648f3f0a7e3c8d5a631de23"
+checksum = "db0e12dd59626cd2543903f1b794135b1f6e0df1003dd3be1071c06961bf6072"
dependencies = [
"ar",
"cargo_toml",
@@ -962,9 +962,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.20"
+version = "4.5.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
+checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f"
dependencies = [
"clap_builder",
"clap_derive",
@@ -972,9 +972,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.20"
+version = "4.5.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
+checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec"
dependencies = [
"anstream",
"anstyle",
@@ -1500,7 +1500,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
- "thiserror",
+ "thiserror 1.0.68",
"time",
"tokio",
"tokio-stream",
@@ -1790,7 +1790,7 @@ dependencies = [
"http 1.1.0",
"hyper 1.5.0",
"hyper-util",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"rustls-native-certs 0.8.0",
"rustls-pki-types",
"tokio",
@@ -2361,7 +2361,7 @@ dependencies = [
"serde",
"serde_json",
"socket2",
- "thiserror",
+ "thiserror 1.0.68",
"tokio",
"tokio-rustls 0.25.0",
"tokio-util",
@@ -2401,7 +2401,7 @@ dependencies = [
"sha2",
"smallvec",
"subprocess",
- "thiserror",
+ "thiserror 1.0.68",
"uuid",
"zstd",
]
@@ -2893,9 +2893,9 @@ dependencies = [
[[package]]
name = "pgwire"
-version = "0.26.0"
+version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99e0f273b9ffa92a06b0a900c012df432de901c1854b2411cd7b27e2db165cc8"
+checksum = "f8e3b217978f9e224cfd5e2b272064067e793a39744030e49657c699752473c8"
dependencies = [
"async-trait",
"base64 0.22.1",
@@ -2911,7 +2911,7 @@ dependencies = [
"ring",
"rust_decimal",
"stringprep",
- "thiserror",
+ "thiserror 2.0.3",
"tokio",
"tokio-rustls 0.26.0",
"tokio-util",
@@ -3044,7 +3044,7 @@ dependencies = [
"anyhow",
"futures-util",
"pt",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"ssh2",
"tokio",
"tokio-postgres",
@@ -3270,9 +3270,9 @@ dependencies = [
"quinn-proto",
"quinn-udp",
"rustc-hash 2.0.0",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"socket2",
- "thiserror",
+ "thiserror 1.0.68",
"tokio",
"tracing",
]
@@ -3287,9 +3287,9 @@ dependencies = [
"rand",
"ring",
"rustc-hash 2.0.0",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"slab",
- "thiserror",
+ "thiserror 1.0.68",
"tinyvec",
"tracing",
]
@@ -3412,7 +3412,7 @@ dependencies = [
"log",
"regex",
"siphasher 1.0.1",
- "thiserror",
+ "thiserror 1.0.68",
"time",
"tokio",
"tokio-postgres",
@@ -3518,7 +3518,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite",
"quinn",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"rustls-pemfile 2.2.0",
"rustls-pki-types",
"serde",
@@ -3583,9 +3583,9 @@ dependencies = [
[[package]]
name = "rsa"
-version = "0.9.6"
+version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc"
+checksum = "47c75d7c5c6b673e58bf54d8544a9f432e3a925b0e80f7cd3602ab5c50c55519"
dependencies = [
"const-oid",
"digest",
@@ -3686,9 +3686,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.16"
+version = "0.23.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e"
+checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1"
dependencies = [
"log",
"once_cell",
@@ -3916,9 +3916,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.132"
+version = "1.0.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
+checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
dependencies = [
"itoa",
"memchr",
@@ -4029,7 +4029,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085"
dependencies = [
"num-bigint",
"num-traits",
- "thiserror",
+ "thiserror 1.0.68",
"time",
]
@@ -4245,7 +4245,16 @@ version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892"
dependencies = [
- "thiserror-impl",
+ "thiserror-impl 1.0.68",
+]
+
+[[package]]
+name = "thiserror"
+version = "2.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa"
+dependencies = [
+ "thiserror-impl 2.0.3",
]
[[package]]
@@ -4259,6 +4268,17 @@ dependencies = [
"syn 2.0.87",
]
+[[package]]
+name = "thiserror-impl"
+version = "2.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.87",
+]
+
[[package]]
name = "thread_local"
version = "1.1.8"
@@ -4411,7 +4431,7 @@ checksum = "27d684bad428a0f2481f42241f821db42c54e2dc81d8c00db8536c506b0a0144"
dependencies = [
"const-oid",
"ring",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"tokio",
"tokio-postgres",
"tokio-rustls 0.26.0",
@@ -4445,7 +4465,7 @@ version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
dependencies = [
- "rustls 0.23.16",
+ "rustls 0.23.19",
"rustls-pki-types",
"tokio",
]
@@ -4630,9 +4650,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
-version = "0.1.40"
+version = "0.1.41"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
dependencies = [
"pin-project-lite",
"tracing-attributes",
@@ -4646,16 +4666,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf"
dependencies = [
"crossbeam-channel",
- "thiserror",
+ "thiserror 1.0.68",
"time",
"tracing-subscriber",
]
[[package]]
name = "tracing-attributes"
-version = "0.1.27"
+version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
+checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d"
dependencies = [
"proc-macro2",
"quote",
@@ -4664,9 +4684,9 @@ dependencies = [
[[package]]
name = "tracing-core"
-version = "0.1.32"
+version = "0.1.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
dependencies = [
"once_cell",
"valuable",
@@ -4685,9 +4705,9 @@ dependencies = [
[[package]]
name = "tracing-subscriber"
-version = "0.3.18"
+version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
+checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008"
dependencies = [
"matchers",
"nu-ansi-term",
@@ -4781,21 +4801,24 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
-version = "2.10.1"
+version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a"
+checksum = "b30e6f97efe1fa43535ee241ee76967d3ff6ff3953ebb430d8d55c5393029e7b"
dependencies = [
"base64 0.22.1",
"encoding_rs",
"flate2",
+ "litemap",
"log",
"once_cell",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"rustls-pki-types",
"serde",
"serde_json",
"url",
"webpki-roots",
+ "yoke",
+ "zerofrom",
]
[[package]]
@@ -5285,9 +5308,9 @@ dependencies = [
[[package]]
name = "x509-certificate"
-version = "0.23.1"
+version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66534846dec7a11d7c50a74b7cdb208b9a581cad890b7866430d438455847c85"
+checksum = "e57b9f8bcae7c1f36479821ae826d75050c60ce55146fd86d3553ed2573e2762"
dependencies = [
"bcder",
"bytes",
@@ -5298,7 +5321,7 @@ dependencies = [
"ring",
"signature",
"spki",
- "thiserror",
+ "thiserror 1.0.68",
"zeroize",
]
@@ -5358,7 +5381,7 @@ dependencies = [
"hyper-util",
"log",
"percent-encoding",
- "rustls 0.23.16",
+ "rustls 0.23.19",
"rustls-pemfile 2.2.0",
"seahash",
"serde",
diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml
index 6efea5f4b2..5f5d1b3e6a 100644
--- a/nexus/Cargo.toml
+++ b/nexus/Cargo.toml
@@ -32,7 +32,7 @@ ssh2 = "0.9"
sqlparser = { git = "https://github.com/peerdb-io/sqlparser-rs.git", branch = "main" }
tokio = { version = "1", features = ["full"] }
tracing = "0.1"
-pgwire = { version = "0.26", default-features = false, features = [
+pgwire = { version = "0.27", default-features = false, features = [
"scram",
"server-api-ring",
] }
diff --git a/nexus/analyzer/src/lib.rs b/nexus/analyzer/src/lib.rs
index 830da627de..cc9309c6b3 100644
--- a/nexus/analyzer/src/lib.rs
+++ b/nexus/analyzer/src/lib.rs
@@ -48,7 +48,7 @@ pub enum QueryAssociation {
Catalog,
}
-impl<'a> StatementAnalyzer for PeerExistanceAnalyzer<'a> {
+impl StatementAnalyzer for PeerExistanceAnalyzer<'_> {
type Output = QueryAssociation;
fn analyze(&self, statement: &Statement) -> anyhow::Result