From ccb47c015ddfbd41b403967d62726e2dffb08180 Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Sat, 24 Aug 2024 15:12:11 +0300 Subject: [PATCH] fix: fixed transformers threshold validation * fixed case when the min and max thresholds provided were ignored --- .../postgres/transformers/noise_date_test.go | 16 +++++ .../db/postgres/transformers/noise_float.go | 45 ++++++++------ .../postgres/transformers/noise_float_test.go | 13 +++++ .../db/postgres/transformers/noise_int.go | 24 ++++++-- .../db/postgres/transformers/noise_numeric.go | 47 ++++++++++----- .../transformers/noise_numeric_test.go | 13 +++++ .../db/postgres/transformers/random_float.go | 20 +++++-- .../transformers/random_float_test.go | 15 +++++ .../db/postgres/transformers/random_int.go | 41 ++++++++----- .../postgres/transformers/random_int_test.go | 58 +++++++++++++++++-- .../postgres/transformers/random_numeric.go | 40 +++++++++---- .../transformers/random_numeric_test.go | 14 +++++ 12 files changed, 274 insertions(+), 72 deletions(-) diff --git a/internal/db/postgres/transformers/noise_date_test.go b/internal/db/postgres/transformers/noise_date_test.go index 0d81fdf0..af84f361 100644 --- a/internal/db/postgres/transformers/noise_date_test.go +++ b/internal/db/postgres/transformers/noise_date_test.go @@ -93,6 +93,22 @@ func TestNoiseDateTransformer_Transform(t *testing.T) { max: time.Date(2024, 8, 29, 1, 1, 1, 1000, loc), }, }, + { + name: "test timestamp type with Truncate till day", + params: map[string]toolkit.ParamsValue{ + "max_ratio": toolkit.ParamsValue("1 year 1 mons 1 day 01:01:01.01"), + "truncate": toolkit.ParamsValue("month"), + "column": toolkit.ParamsValue("date_ts"), + "min": toolkit.ParamsValue("2022-06-01 22:00:00"), + "max": toolkit.ParamsValue("2024-01-29 01:01:01.001"), + }, + original: "2023-06-25 00:00:00", + result: result{ + pattern: `^\d{4}-\d{2}-01 0{2}:0{2}:0{2}$`, + min: time.Date(2022, 3, 1, 22, 00, 0, 0, loc), + max: time.Date(2024, 8, 29, 1, 1, 1, 1000, loc), + }, + }, } for _, tt := range tests { diff --git a/internal/db/postgres/transformers/noise_float.go b/internal/db/postgres/transformers/noise_float.go index d55e5067..9c8be144 100644 --- a/internal/db/postgres/transformers/noise_float.go +++ b/internal/db/postgres/transformers/noise_float.go @@ -17,7 +17,6 @@ package transformers import ( "context" "fmt" - "math" "github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils" "github.com/greenmaskio/greenmask/internal/generators/transformers" @@ -97,7 +96,8 @@ func NewNoiseFloatTransformer(ctx context.Context, driver *toolkit.Driver, param var columnName, engine string var dynamicMode bool - var minValueThreshold, maxValueThreshold, minRatio, maxRatio float64 + var minValueThreshold, maxValueThreshold *float64 + var minRatio, maxRatio float64 var decimal int columnParam := parameters["column"] @@ -129,11 +129,23 @@ func NewNoiseFloatTransformer(ctx context.Context, driver *toolkit.Driver, param floatSize := c.GetColumnSize() if !dynamicMode { - if err := minParam.Scan(&minValueThreshold); err != nil { - return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + minIsEmpty, err := minParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err) + } + if !minIsEmpty { + if err = minParam.Scan(&minValueThreshold); err != nil { + return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + } } - if err := maxParam.Scan(&maxValueThreshold); err != nil { - return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + maxIsEmpty, err := maxParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err) + } + if !maxIsEmpty { + if err = maxParam.Scan(&maxValueThreshold); err != nil { + return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + } } } @@ -244,15 +256,21 @@ func (nft *NoiseFloatTransformer) Transform(ctx context.Context, r *toolkit.Reco } func validateNoiseFloatTypeAndSetLimit( - size int, requestedMinValue, requestedMaxValue float64, decimal int, + size int, requestedMinValue, requestedMaxValue *float64, decimal int, ) (limiter *transformers.NoiseFloat64Limiter, warns toolkit.ValidationWarnings, err error) { minValue, maxValue, err := getFloatThresholds(size) if err != nil { return nil, nil, err } + if requestedMinValue == nil { + requestedMinValue = &minValue + } + if requestedMaxValue == nil { + requestedMaxValue = &maxValue + } - if !limitIsValid(requestedMinValue, minValue, maxValue) { + if !limitIsValid(*requestedMinValue, minValue, maxValue) { warns = append(warns, toolkit.NewValidationWarning(). SetMsgf("requested min value is out of float%d range", size). SetSeverity(toolkit.ErrorValidationSeverity). @@ -263,7 +281,7 @@ func validateNoiseFloatTypeAndSetLimit( ) } - if !limitIsValid(requestedMaxValue, minValue, maxValue) { + if !limitIsValid(*requestedMaxValue, minValue, maxValue) { warns = append(warns, toolkit.NewValidationWarning(). SetMsgf("requested max value is out of float%d range", size). SetSeverity(toolkit.ErrorValidationSeverity). @@ -278,18 +296,11 @@ func validateNoiseFloatTypeAndSetLimit( return nil, warns, nil } - limiter, err = transformers.NewNoiseFloat64Limiter(-math.MaxFloat64, math.MaxFloat64, decimal) + limiter, err = transformers.NewNoiseFloat64Limiter(*requestedMinValue, *requestedMaxValue, decimal) if err != nil { return nil, nil, err } - if requestedMinValue != 0 || requestedMaxValue != 0 { - limiter, err = transformers.NewNoiseFloat64Limiter(requestedMinValue, requestedMaxValue, decimal) - if err != nil { - return nil, nil, err - } - } - return limiter, nil, nil } diff --git a/internal/db/postgres/transformers/noise_float_test.go b/internal/db/postgres/transformers/noise_float_test.go index 443df76a..9024d1a2 100644 --- a/internal/db/postgres/transformers/noise_float_test.go +++ b/internal/db/postgres/transformers/noise_float_test.go @@ -107,6 +107,19 @@ func TestNoiseFloatTransformer_Transform(t *testing.T) { input: "100", result: result{min: 90, max: 110, regexp: `^-*\d+$`}, }, + { + name: "with thresholds min zero", + columnName: "col_float8", + params: map[string]toolkit.ParamsValue{ + "min_ratio": toolkit.ParamsValue("0.2"), + "max_ratio": toolkit.ParamsValue("0.9"), + "min": toolkit.ParamsValue("0"), + "max": toolkit.ParamsValue("110"), + "decimal": toolkit.ParamsValue("0"), + }, + input: "100", + result: result{min: 0, max: 110, regexp: `^-*\d+$`}, + }, } for _, tt := range tests { diff --git a/internal/db/postgres/transformers/noise_int.go b/internal/db/postgres/transformers/noise_int.go index 8cc6474b..91d5cd9c 100644 --- a/internal/db/postgres/transformers/noise_int.go +++ b/internal/db/postgres/transformers/noise_int.go @@ -86,7 +86,7 @@ type NoiseIntTransformer struct { func NewNoiseIntTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) { var columnName, engine string var minRatio, maxRatio float64 - var maxValueThreshold, minValueThreshold int64 + var maxValueThreshold, minValueThreshold *int64 var dynamicMode bool columnParam := parameters["column"] @@ -118,11 +118,23 @@ func NewNoiseIntTransformer(ctx context.Context, driver *toolkit.Driver, paramet } if !dynamicMode { - if err := minParam.Scan(&maxValueThreshold); err != nil { - return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + minIsEmpty, err := minParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err) + } + if !minIsEmpty { + if err = minParam.Scan(&minValueThreshold); err != nil { + return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + } + } + maxIsEmpty, err := maxParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err) } - if err := maxParam.Scan(&minValueThreshold); err != nil { - return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + if !maxIsEmpty { + if err = maxParam.Scan(&maxValueThreshold); err != nil { + return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + } } } @@ -227,7 +239,7 @@ func (nit *NoiseIntTransformer) Transform(ctx context.Context, r *toolkit.Record } func validateIntTypeAndSetNoiseInt64Limiter( - size int, requestedMinValue, requestedMaxValue int64, + size int, requestedMinValue, requestedMaxValue *int64, ) (limiter *transformers.NoiseInt64Limiter, warns toolkit.ValidationWarnings, err error) { minValue, maxValue, warns, err := validateInt64AndGetLimits(size, requestedMinValue, requestedMaxValue) diff --git a/internal/db/postgres/transformers/noise_numeric.go b/internal/db/postgres/transformers/noise_numeric.go index b57e16d9..7fd14f6e 100644 --- a/internal/db/postgres/transformers/noise_numeric.go +++ b/internal/db/postgres/transformers/noise_numeric.go @@ -118,7 +118,7 @@ func NewNumericFloatTransformer(ctx context.Context, driver *toolkit.Driver, par var columnName, engine string var dynamicMode bool var minRatio, maxRatio float64 - var minValueThreshold, maxValueThreshold decimal.Decimal + var minValueThreshold, maxValueThreshold *decimal.Decimal var precision int32 columnParam := parameters["column"] @@ -149,14 +149,34 @@ func NewNumericFloatTransformer(ctx context.Context, driver *toolkit.Driver, par affectedColumns[idx] = columnName if !dynamicMode { - if err := minParam.Scan(&minValueThreshold); err != nil { - return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + minIsEmpty, err := minParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err) + } + if !minIsEmpty { + if err = minParam.Scan(&minValueThreshold); err != nil { + return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + } + } + maxIsEmpty, err := maxParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err) } - if err := maxParam.Scan(&maxValueThreshold); err != nil { - return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + if !maxIsEmpty { + if err = maxParam.Scan(&maxValueThreshold); err != nil { + return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + } } } + limiter, limitsWarnings, err := validateNoiseNumericTypeAndSetLimit(bigIntegerTransformerGenByteLength, minValueThreshold, maxValueThreshold) + if err != nil { + return nil, nil, err + } + if limitsWarnings.IsFatal() { + return nil, limitsWarnings, nil + } + if err := decimalParam.Scan(&precision); err != nil { return nil, nil, fmt.Errorf(`unable to scan "decimal" param: %w`, err) } @@ -169,13 +189,6 @@ func NewNumericFloatTransformer(ctx context.Context, driver *toolkit.Driver, par return nil, nil, fmt.Errorf("unable to scan \"max_ratio\" param: %w", err) } - limiter, limitsWarnings, err := validateNoiseNumericTypeAndSetLimit(bigIntegerTransformerGenByteLength, minValueThreshold, maxValueThreshold) - if err != nil { - return nil, nil, err - } - if limitsWarnings.IsFatal() { - return nil, limitsWarnings, nil - } limiter.SetPrecision(precision) t := transformers.NewNoiseNumericTransformer(limiter, minRatio, maxRatio) @@ -269,7 +282,7 @@ func (nft *NoiseNumericTransformer) Transform(ctx context.Context, r *toolkit.Re } func validateNoiseNumericTypeAndSetLimit( - size int, requestedMinValue, requestedMaxValue decimal.Decimal, + size int, requestedMinValue, requestedMaxValue *decimal.Decimal, ) (limiter *transformers.NoiseNumericLimiter, warns toolkit.ValidationWarnings, err error) { minVal, maxVal, warns, err := getNumericThresholds(size, requestedMinValue, requestedMaxValue) @@ -279,8 +292,14 @@ func validateNoiseNumericTypeAndSetLimit( if warns.IsFatal() { return nil, warns, nil } + if requestedMinValue == nil { + requestedMinValue = &minVal + } + if requestedMaxValue == nil { + requestedMaxValue = &maxVal + } - limiter, err = transformers.NewNoiseNumericLimiter(minVal, maxVal) + limiter, err = transformers.NewNoiseNumericLimiter(*requestedMinValue, *requestedMaxValue) if err != nil { return nil, nil, fmt.Errorf("error creating limiter by size: %w", err) } diff --git a/internal/db/postgres/transformers/noise_numeric_test.go b/internal/db/postgres/transformers/noise_numeric_test.go index 86ba4ced..61600277 100644 --- a/internal/db/postgres/transformers/noise_numeric_test.go +++ b/internal/db/postgres/transformers/noise_numeric_test.go @@ -97,6 +97,19 @@ func TestNoiseNumericTransformer_Transform(t *testing.T) { input: "100", result: result{min: 90, max: 110, regexp: `^-*\d+$`}, }, + { + name: "numeric with thresholds", + columnName: "id_numeric", + params: map[string]toolkit.ParamsValue{ + "min_ratio": toolkit.ParamsValue("0.2"), + "max_ratio": toolkit.ParamsValue("0.9"), + "min": toolkit.ParamsValue("0"), + "max": toolkit.ParamsValue("50"), + "decimal": toolkit.ParamsValue("4"), + }, + input: "100", + result: result{min: 10, max: 190, regexp: `^-*\d+[.]*\d{0,4}$`}, + }, } for _, tt := range tests { diff --git a/internal/db/postgres/transformers/random_float.go b/internal/db/postgres/transformers/random_float.go index 366a6b67..b6cdd693 100644 --- a/internal/db/postgres/transformers/random_float.go +++ b/internal/db/postgres/transformers/random_float.go @@ -138,11 +138,23 @@ func NewFloatTransformer(ctx context.Context, driver *toolkit.Driver, parameters } if !dynamicMode { - if err := minParam.Scan(&minVal); err != nil { - return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + minIsEmpty, err := minParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err) + } + if !minIsEmpty { + if err = minParam.Scan(&minVal); err != nil { + return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + } + } + maxIsEmpty, err := maxParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err) } - if err := maxParam.Scan(&maxVal); err != nil { - return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + if !maxIsEmpty { + if err = maxParam.Scan(&maxVal); err != nil { + return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + } } } diff --git a/internal/db/postgres/transformers/random_float_test.go b/internal/db/postgres/transformers/random_float_test.go index a519366e..dfd7cb7a 100644 --- a/internal/db/postgres/transformers/random_float_test.go +++ b/internal/db/postgres/transformers/random_float_test.go @@ -121,6 +121,21 @@ func TestRandomFloatTransformer_Transform(t *testing.T) { isNull: true, }, }, + { + name: "keep_null true and NULL seq", + columnName: "col_float8", + originalValue: "\\N", + params: map[string]toolkit.ParamsValue{ + "min": toolkit.ParamsValue("0"), + "max": toolkit.ParamsValue("1000"), + "decimal": toolkit.ParamsValue("0"), + "keep_null": toolkit.ParamsValue("false"), + }, + result: result{ + min: 0, + max: 1000, + }, + }, //{ // name: "text with default float8", // params: map[string]toolkit.ParamsValue{ diff --git a/internal/db/postgres/transformers/random_int.go b/internal/db/postgres/transformers/random_int.go index 9d5b7e30..5982edfb 100644 --- a/internal/db/postgres/transformers/random_int.go +++ b/internal/db/postgres/transformers/random_int.go @@ -93,7 +93,7 @@ type IntegerTransformer struct { func NewIntegerTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) { var columnName, engine string - var minVal, maxVal int64 + var minVal, maxVal *int64 var keepNull, dynamicMode bool columnParam := parameters["column"] @@ -127,11 +127,23 @@ func NewIntegerTransformer(ctx context.Context, driver *toolkit.Driver, paramete } if !dynamicMode { - if err := minParam.Scan(&minVal); err != nil { - return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + minIsEmpty, err := minParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err) + } + if !minIsEmpty { + if err = minParam.Scan(&minVal); err != nil { + return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + } } - if err := maxParam.Scan(&maxVal); err != nil { - return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + maxIsEmpty, err := maxParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err) + } + if !maxIsEmpty { + if err = maxParam.Scan(&maxVal); err != nil { + return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + } } } @@ -281,7 +293,7 @@ func limitIsValid[T int64 | float64](requestedThreshold, minValue, maxValue T) b } func validateIntTypeAndSetRandomInt64Limiter( - size int, requestedMinValue, requestedMaxValue int64, + size int, requestedMinValue, requestedMaxValue *int64, ) (limiter *transformers.Int64Limiter, warns toolkit.ValidationWarnings, err error) { minValue, maxValue, warns, err := validateInt64AndGetLimits(size, requestedMinValue, requestedMaxValue) @@ -299,15 +311,21 @@ func validateIntTypeAndSetRandomInt64Limiter( } func validateInt64AndGetLimits( - size int, requestedMinValue, requestedMaxValue int64, + size int, requestedMinValue, requestedMaxValue *int64, ) (int64, int64, toolkit.ValidationWarnings, error) { var warns toolkit.ValidationWarnings minValue, maxValue, err := getIntThresholds(size) if err != nil { return 0, 0, nil, err } + if requestedMinValue == nil { + requestedMinValue = &minValue + } + if requestedMaxValue == nil { + requestedMaxValue = &maxValue + } - if !limitIsValid(requestedMinValue, minValue, maxValue) { + if !limitIsValid(*requestedMinValue, minValue, maxValue) { warns = append(warns, toolkit.NewValidationWarning(). SetMsgf("requested min value is out of int%d range", size). SetSeverity(toolkit.ErrorValidationSeverity). @@ -318,7 +336,7 @@ func validateInt64AndGetLimits( ) } - if !limitIsValid(requestedMaxValue, minValue, maxValue) { + if !limitIsValid(*requestedMaxValue, minValue, maxValue) { warns = append(warns, toolkit.NewValidationWarning(). SetMsgf("requested max value is out of int%d range", size). SetSeverity(toolkit.ErrorValidationSeverity). @@ -332,11 +350,8 @@ func validateInt64AndGetLimits( if warns.IsFatal() { return 0, 0, warns, nil } - if requestedMinValue != 0 || requestedMaxValue != 0 { - return requestedMinValue, requestedMaxValue, nil, nil - } - return minValue, maxValue, nil, nil + return *requestedMinValue, *requestedMaxValue, nil, nil } func init() { diff --git a/internal/db/postgres/transformers/random_int_test.go b/internal/db/postgres/transformers/random_int_test.go index 5352df9d..e009c84e 100644 --- a/internal/db/postgres/transformers/random_int_test.go +++ b/internal/db/postgres/transformers/random_int_test.go @@ -13,12 +13,19 @@ import ( func TestRandomIntTransformer_Transform_random_static(t *testing.T) { + type expected struct { + min int64 + max int64 + isNull bool + } + tests := []struct { name string columnName string originalValue string expectedRegexp string params map[string]toolkit.ParamsValue + expected expected }{ { name: "int2", @@ -29,6 +36,10 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { "min": toolkit.ParamsValue("1"), "max": toolkit.ParamsValue("100"), }, + expected: expected{ + min: 1, + max: 100, + }, }, { name: "int4", @@ -39,6 +50,10 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { "min": toolkit.ParamsValue("1"), "max": toolkit.ParamsValue("100"), }, + expected: expected{ + min: 1, + max: 100, + }, }, { name: "int8", @@ -49,6 +64,10 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { "min": toolkit.ParamsValue("1"), "max": toolkit.ParamsValue("100"), }, + expected: expected{ + min: 1, + max: 100, + }, }, { name: "keep_null false and NULL seq", @@ -60,6 +79,10 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { "max": toolkit.ParamsValue("100"), "keep_null": toolkit.ParamsValue("false"), }, + expected: expected{ + min: 1, + max: 100, + }, }, { name: "keep_null true and NULL seq", @@ -72,6 +95,27 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { "keep_null": toolkit.ParamsValue("true"), "engine": toolkit.ParamsValue("random"), }, + expected: expected{ + min: 1, + max: 100, + isNull: true, + }, + }, + { + name: "test zero min", + columnName: "id8", + originalValue: "\\N", + expectedRegexp: fmt.Sprintf(`^(\%s)$`, "\\N"), + params: map[string]toolkit.ParamsValue{ + "min": toolkit.ParamsValue("0"), + "max": toolkit.ParamsValue("100"), + "engine": toolkit.ParamsValue("random"), + "keep_null": toolkit.ParamsValue("false"), + }, + expected: expected{ + min: 0, + max: 100, + }, }, } @@ -96,12 +140,14 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { record, ) require.NoError(t, err) - - encoded, err := r.Encode() - require.NoError(t, err) - res, err := encoded.Encode() - require.NoError(t, err) - require.Regexp(t, tt.expectedRegexp, string(res)) + rawData, _ := r.GetRawColumnValueByName(tt.columnName) + require.Equal(t, tt.expected.isNull, rawData.IsNull) + if !rawData.IsNull { + var resInt int64 + _, err = r.ScanColumnValueByName(tt.columnName, &resInt) + require.NoError(t, err) + require.True(t, resInt >= tt.expected.min && resInt <= tt.expected.max) + } }) } diff --git a/internal/db/postgres/transformers/random_numeric.go b/internal/db/postgres/transformers/random_numeric.go index 7199c6e8..cb73d95c 100644 --- a/internal/db/postgres/transformers/random_numeric.go +++ b/internal/db/postgres/transformers/random_numeric.go @@ -106,7 +106,7 @@ type NumericTransformer struct { func NewRandomNumericTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) { var columnName, engine string - var minVal, maxVal decimal.Decimal + var minVal, maxVal *decimal.Decimal var keepNull, dynamicMode bool var precision int32 @@ -144,12 +144,25 @@ func NewRandomNumericTransformer(ctx context.Context, driver *toolkit.Driver, pa } if !dynamicMode { - if err := minParam.Scan(&minVal); err != nil { - return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + minIsEmpty, err := minParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err) } - if err := maxParam.Scan(&maxVal); err != nil { - return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + if !minIsEmpty { + if err = minParam.Scan(&minVal); err != nil { + return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err) + } + } + maxIsEmpty, err := maxParam.IsEmpty() + if err != nil { + return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err) } + if !maxIsEmpty { + if err = maxParam.Scan(&maxVal); err != nil { + return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err) + } + } + } limiter, limitsWarnings, err := validateRandomNumericTypeAndSetLimit(bigIntegerTransformerGenByteLength, minVal, maxVal) @@ -250,7 +263,7 @@ func (bit *NumericTransformer) Transform(ctx context.Context, r *toolkit.Record) return r, nil } -func getNumericThresholds(size int, requestedMinValue, requestedMaxValue decimal.Decimal, +func getNumericThresholds(size int, requestedMinValue, requestedMaxValue *decimal.Decimal, ) (decimal.Decimal, decimal.Decimal, toolkit.ValidationWarnings, error) { var warns toolkit.ValidationWarnings minVal, maxVal, err := transformers.GetMinAndMaxNumericValueBySetting(size) @@ -258,11 +271,14 @@ func getNumericThresholds(size int, requestedMinValue, requestedMaxValue decimal return decimal.Decimal{}, decimal.Decimal{}, nil, fmt.Errorf("error creating limiter by size: %w", err) } - if requestedMinValue.Equal(decimal.NewFromInt(0)) && requestedMinValue.Equal(decimal.NewFromInt(0)) { - return minVal, maxVal, nil, nil + if requestedMinValue == nil { + requestedMinValue = &minVal + } + if requestedMaxValue == nil { + requestedMaxValue = &maxVal } - if !numericLimitIsValid(requestedMinValue, minVal, maxVal) { + if !numericLimitIsValid(*requestedMinValue, minVal, maxVal) { warns = append(warns, toolkit.NewValidationWarning(). SetMsgf("requested min value is out of numeric(%d) range", size). SetSeverity(toolkit.ErrorValidationSeverity). @@ -273,7 +289,7 @@ func getNumericThresholds(size int, requestedMinValue, requestedMaxValue decimal ) } - if !numericLimitIsValid(requestedMaxValue, minVal, maxVal) { + if !numericLimitIsValid(*requestedMaxValue, minVal, maxVal) { warns = append(warns, toolkit.NewValidationWarning(). SetMsgf("requested max value is out of NEMERIC(%d) range", size). SetSeverity(toolkit.ErrorValidationSeverity). @@ -286,11 +302,11 @@ func getNumericThresholds(size int, requestedMinValue, requestedMaxValue decimal if warns.IsFatal() { return decimal.Decimal{}, decimal.Decimal{}, warns, nil } - return requestedMinValue, requestedMaxValue, nil, nil + return *requestedMinValue, *requestedMaxValue, nil, nil } func validateRandomNumericTypeAndSetLimit( - size int, requestedMinValue, requestedMaxValue decimal.Decimal, + size int, requestedMinValue, requestedMaxValue *decimal.Decimal, ) (limiter *transformers.RandomNumericLimiter, warns toolkit.ValidationWarnings, err error) { minVal, maxVal, warns, err := getNumericThresholds(size, requestedMinValue, requestedMaxValue) diff --git a/internal/db/postgres/transformers/random_numeric_test.go b/internal/db/postgres/transformers/random_numeric_test.go index 429fe86e..56d87d19 100644 --- a/internal/db/postgres/transformers/random_numeric_test.go +++ b/internal/db/postgres/transformers/random_numeric_test.go @@ -67,6 +67,20 @@ func TestBigIntTransformer_Transform_random_static(t *testing.T) { isNull: true, }, }, + { + name: "Regression for implicitly set threshold", + columnName: "id_numeric", + originalValue: "12345", + params: map[string]toolkit.ParamsValue{ + "min": toolkit.ParamsValue("0.0"), + "max": toolkit.ParamsValue("10.0"), + "decimal": toolkit.ParamsValue("2"), + }, + expected: expected{ + min: decimal.RequireFromString("0.0"), + max: decimal.RequireFromString("10.0"), + }, + }, } for _, tt := range tests {