Skip to content

Commit

Permalink
env: PEERDB_SNOWFLAKE_ON_ERROR (#2189)
Browse files Browse the repository at this point in the history
not wiring as a dynamic config as if exposed in UI I'd want to put validation on it

https://docs.snowflake.com/en/sql-reference/sql/copy-into-table#copy-options-copyoptions
  • Loading branch information
serprex authored Oct 25, 2024
1 parent edcb8f5 commit 323bb69
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions flow/connectors/snowflake/qrep_avro_consolidate.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/PeerDB-io/peer-flow/connectors/utils"
"github.com/PeerDB-io/peer-flow/generated/protos"
"github.com/PeerDB-io/peer-flow/peerdbenv"
"github.com/PeerDB-io/peer-flow/shared"
)

Expand Down Expand Up @@ -136,8 +137,12 @@ func (s *SnowflakeAvroConsolidateHandler) getCopyTransformation(copyDstTable str
s.config.SyncedAtColName,
s.config.SoftDeleteColName,
)
return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE",
copyDstTable, columnsSQL, transformationSQL, s.stage)
onErrorStr := ""
if onError := peerdbenv.GetEnvString("PEERDB_SNOWFLAKE_ON_ERROR", ""); onError != "" {
onErrorStr = ", ON_ERROR=" + onError
}
return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE%s",
copyDstTable, columnsSQL, transformationSQL, s.stage, onErrorStr)
}

func (s *SnowflakeAvroConsolidateHandler) handleAppendMode(ctx context.Context) error {
Expand Down

0 comments on commit 323bb69

Please sign in to comment.