refactor: reduce default batch and partition sizes for improved stabi…

…lity

Reduces default batch sizes from 1,000,000 to 250,000 rows across CDC and snapshot operations:
- Lower CDC batch size for incremental syncs
- Reduce snapshot partition size for initial loads
- Update UI defaults and tooltips to reflect new values

This change helps prevent memory pressure and timeout issues when processing large datasets.

flow/activities/flowable_core.go

@@ -139,7 +139,7 @@ func syncCore[TPull connectors.CDCPullConnectorCore, TSync connectors.CDCSyncCon
 
 	batchSize := options.BatchSize
 	if batchSize == 0 {
-		batchSize = 1_000_000
+		batchSize = 250_000
 	}
 
 	lastOffset, err := func() (int64, error) {

flow/workflows/snapshot_flow.go

@@ -166,7 +166,7 @@ func (s *SnapshotFlowExecution) cloneTable(
 		numWorkers = s.config.SnapshotMaxParallelWorkers
 	}
 
-	numRowsPerPartition := uint32(500000)
+	numRowsPerPartition := uint32(250000)
 	if s.config.SnapshotNumRowsPerPartition > 0 {
 		numRowsPerPartition = s.config.SnapshotNumRowsPerPartition
 	}

ui/app/mirrors/create/helpers/cdc.ts

@@ -22,12 +22,12 @@ export const cdcSettings: MirrorSetting[] = [
       setter(
         (curr: CDCConfig): CDCConfig => ({
           ...curr,
-          maxBatchSize: (value as number) || 1000000,
+          maxBatchSize: (value as number) || 250000,
         })
       ),
-    tips: 'The number of rows PeerDB will pull from source at a time. If left empty, the default value is 1,000,000 rows.',
+    tips: 'The number of rows PeerDB will pull from source at a time. If left empty, the default value is 250,000 rows.',
     type: 'number',
-    default: '1000000',
+    default: '250000',
     advanced: AdvancedSettingType.ALL,
   },
   {
@@ -78,11 +78,11 @@ export const cdcSettings: MirrorSetting[] = [
       setter(
         (curr: CDCConfig): CDCConfig => ({
           ...curr,
-          snapshotNumRowsPerPartition: parseInt(value as string, 10) || 1000000,
+          snapshotNumRowsPerPartition: parseInt(value as string, 10) || 250000,
         })
       ),
-    tips: 'PeerDB splits up table data into partitions for increased performance. This setting controls the number of rows per partition. The default value is 1000000.',
-    default: '1000000',
+    tips: 'PeerDB splits up table data into partitions for increased performance. This setting controls the number of rows per partition. The default value is 250000.',
+    default: '250000',
     type: 'number',
     advanced: AdvancedSettingType.ALL,
   },

ui/app/mirrors/create/helpers/common.ts

@@ -25,10 +25,10 @@ export const blankCDCSetting: CDCConfig = {
   destinationName: '',
   flowJobName: '',
   tableMappings: [],
-  maxBatchSize: 1000000,
+  maxBatchSize: 250000,
   doInitialSnapshot: true,
   publicationName: '',
-  snapshotNumRowsPerPartition: 1000000,
+  snapshotNumRowsPerPartition: 250000,
   snapshotMaxParallelWorkers: 4,
   snapshotNumTablesInParallel: 1,
   snapshotStagingPath: '',