Skip to content

Commit

Permalink
fix: fixed transformers threshold validation
Browse files Browse the repository at this point in the history 
* fixed case when the min and max thresholds provided were ignored
  • Loading branch information
@wwoytenko
wwoytenko committed Oct 13, 2024
1 parent ab15ebb commit ccb47c0
Show file tree
Hide file tree
Showing 12 changed files with 274 additions and 72 deletions.
16 changes: 16 additions & 0 deletions internal/db/postgres/transformers/noise_date_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,22 @@ func TestNoiseDateTransformer_Transform(t *testing.T) {
max: time.Date(2024, 8, 29, 1, 1, 1, 1000, loc),
},
},
{
name: "test timestamp type with Truncate till day",
params: map[string]toolkit.ParamsValue{
"max_ratio": toolkit.ParamsValue("1 year 1 mons 1 day 01:01:01.01"),
"truncate": toolkit.ParamsValue("month"),
"column": toolkit.ParamsValue("date_ts"),
"min": toolkit.ParamsValue("2022-06-01 22:00:00"),
"max": toolkit.ParamsValue("2024-01-29 01:01:01.001"),
},
original: "2023-06-25 00:00:00",
result: result{
pattern: `^\d{4}-\d{2}-01 0{2}:0{2}:0{2}$`,
min: time.Date(2022, 3, 1, 22, 00, 0, 0, loc),
max: time.Date(2024, 8, 29, 1, 1, 1, 1000, loc),
},
},
}

for _, tt := range tests {
Expand Down
45 changes: 28 additions & 17 deletions internal/db/postgres/transformers/noise_float.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ package transformers
import (
"context"
"fmt"
"math"

"github.com/greenmaskio/greenmask/internal/db/postgres/transformers/utils"
"github.com/greenmaskio/greenmask/internal/generators/transformers"
Expand Down Expand Up @@ -97,7 +96,8 @@ func NewNoiseFloatTransformer(ctx context.Context, driver *toolkit.Driver, param

var columnName, engine string
var dynamicMode bool
var minValueThreshold, maxValueThreshold, minRatio, maxRatio float64
var minValueThreshold, maxValueThreshold *float64
var minRatio, maxRatio float64
var decimal int

columnParam := parameters["column"]
Expand Down Expand Up @@ -129,11 +129,23 @@ func NewNoiseFloatTransformer(ctx context.Context, driver *toolkit.Driver, param
floatSize := c.GetColumnSize()

if !dynamicMode {
if err := minParam.Scan(&minValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
minIsEmpty, err := minParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err)
}
if !minIsEmpty {
if err = minParam.Scan(&minValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
}
}
if err := maxParam.Scan(&maxValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
maxIsEmpty, err := maxParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err)
}
if !maxIsEmpty {
if err = maxParam.Scan(&maxValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
}
}
}

Expand Down Expand Up @@ -244,15 +256,21 @@ func (nft *NoiseFloatTransformer) Transform(ctx context.Context, r *toolkit.Reco
}

func validateNoiseFloatTypeAndSetLimit(
size int, requestedMinValue, requestedMaxValue float64, decimal int,
size int, requestedMinValue, requestedMaxValue *float64, decimal int,
) (limiter *transformers.NoiseFloat64Limiter, warns toolkit.ValidationWarnings, err error) {

minValue, maxValue, err := getFloatThresholds(size)
if err != nil {
return nil, nil, err
}
if requestedMinValue == nil {
requestedMinValue = &minValue
}
if requestedMaxValue == nil {
requestedMaxValue = &maxValue
}

if !limitIsValid(requestedMinValue, minValue, maxValue) {
if !limitIsValid(*requestedMinValue, minValue, maxValue) {
warns = append(warns, toolkit.NewValidationWarning().
SetMsgf("requested min value is out of float%d range", size).
SetSeverity(toolkit.ErrorValidationSeverity).
Expand All @@ -263,7 +281,7 @@ func validateNoiseFloatTypeAndSetLimit(
)
}

if !limitIsValid(requestedMaxValue, minValue, maxValue) {
if !limitIsValid(*requestedMaxValue, minValue, maxValue) {
warns = append(warns, toolkit.NewValidationWarning().
SetMsgf("requested max value is out of float%d range", size).
SetSeverity(toolkit.ErrorValidationSeverity).
Expand All @@ -278,18 +296,11 @@ func validateNoiseFloatTypeAndSetLimit(
return nil, warns, nil
}

limiter, err = transformers.NewNoiseFloat64Limiter(-math.MaxFloat64, math.MaxFloat64, decimal)
limiter, err = transformers.NewNoiseFloat64Limiter(*requestedMinValue, *requestedMaxValue, decimal)
if err != nil {
return nil, nil, err
}

if requestedMinValue != 0 || requestedMaxValue != 0 {
limiter, err = transformers.NewNoiseFloat64Limiter(requestedMinValue, requestedMaxValue, decimal)
if err != nil {
return nil, nil, err
}
}

return limiter, nil, nil
}

Expand Down
13 changes: 13 additions & 0 deletions internal/db/postgres/transformers/noise_float_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,19 @@ func TestNoiseFloatTransformer_Transform(t *testing.T) {
input: "100",
result: result{min: 90, max: 110, regexp: `^-*\d+$`},
},
{
name: "with thresholds min zero",
columnName: "col_float8",
params: map[string]toolkit.ParamsValue{
"min_ratio": toolkit.ParamsValue("0.2"),
"max_ratio": toolkit.ParamsValue("0.9"),
"min": toolkit.ParamsValue("0"),
"max": toolkit.ParamsValue("110"),
"decimal": toolkit.ParamsValue("0"),
},
input: "100",
result: result{min: 0, max: 110, regexp: `^-*\d+$`},
},
}

for _, tt := range tests {
Expand Down
24 changes: 18 additions & 6 deletions internal/db/postgres/transformers/noise_int.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ type NoiseIntTransformer struct {
func NewNoiseIntTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) {
var columnName, engine string
var minRatio, maxRatio float64
var maxValueThreshold, minValueThreshold int64
var maxValueThreshold, minValueThreshold *int64
var dynamicMode bool

columnParam := parameters["column"]
Expand Down Expand Up @@ -118,11 +118,23 @@ func NewNoiseIntTransformer(ctx context.Context, driver *toolkit.Driver, paramet
}

if !dynamicMode {
if err := minParam.Scan(&maxValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
minIsEmpty, err := minParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err)
}
if !minIsEmpty {
if err = minParam.Scan(&minValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
}
}
maxIsEmpty, err := maxParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err)
}
if err := maxParam.Scan(&minValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
if !maxIsEmpty {
if err = maxParam.Scan(&maxValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
}
}
}

Expand Down Expand Up @@ -227,7 +239,7 @@ func (nit *NoiseIntTransformer) Transform(ctx context.Context, r *toolkit.Record
}

func validateIntTypeAndSetNoiseInt64Limiter(
size int, requestedMinValue, requestedMaxValue int64,
size int, requestedMinValue, requestedMaxValue *int64,
) (limiter *transformers.NoiseInt64Limiter, warns toolkit.ValidationWarnings, err error) {

minValue, maxValue, warns, err := validateInt64AndGetLimits(size, requestedMinValue, requestedMaxValue)
Expand Down
47 changes: 33 additions & 14 deletions internal/db/postgres/transformers/noise_numeric.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func NewNumericFloatTransformer(ctx context.Context, driver *toolkit.Driver, par
var columnName, engine string
var dynamicMode bool
var minRatio, maxRatio float64
var minValueThreshold, maxValueThreshold decimal.Decimal
var minValueThreshold, maxValueThreshold *decimal.Decimal
var precision int32

columnParam := parameters["column"]
Expand Down Expand Up @@ -149,14 +149,34 @@ func NewNumericFloatTransformer(ctx context.Context, driver *toolkit.Driver, par
affectedColumns[idx] = columnName

if !dynamicMode {
if err := minParam.Scan(&minValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
minIsEmpty, err := minParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err)
}
if !minIsEmpty {
if err = minParam.Scan(&minValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
}
}
maxIsEmpty, err := maxParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err)
}
if err := maxParam.Scan(&maxValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
if !maxIsEmpty {
if err = maxParam.Scan(&maxValueThreshold); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
}
}
}

limiter, limitsWarnings, err := validateNoiseNumericTypeAndSetLimit(bigIntegerTransformerGenByteLength, minValueThreshold, maxValueThreshold)
if err != nil {
return nil, nil, err
}
if limitsWarnings.IsFatal() {
return nil, limitsWarnings, nil
}

if err := decimalParam.Scan(&precision); err != nil {
return nil, nil, fmt.Errorf(`unable to scan "decimal" param: %w`, err)
}
Expand All @@ -169,13 +189,6 @@ func NewNumericFloatTransformer(ctx context.Context, driver *toolkit.Driver, par
return nil, nil, fmt.Errorf("unable to scan \"max_ratio\" param: %w", err)
}

limiter, limitsWarnings, err := validateNoiseNumericTypeAndSetLimit(bigIntegerTransformerGenByteLength, minValueThreshold, maxValueThreshold)
if err != nil {
return nil, nil, err
}
if limitsWarnings.IsFatal() {
return nil, limitsWarnings, nil
}
limiter.SetPrecision(precision)

t := transformers.NewNoiseNumericTransformer(limiter, minRatio, maxRatio)
Expand Down Expand Up @@ -269,7 +282,7 @@ func (nft *NoiseNumericTransformer) Transform(ctx context.Context, r *toolkit.Re
}

func validateNoiseNumericTypeAndSetLimit(
size int, requestedMinValue, requestedMaxValue decimal.Decimal,
size int, requestedMinValue, requestedMaxValue *decimal.Decimal,
) (limiter *transformers.NoiseNumericLimiter, warns toolkit.ValidationWarnings, err error) {

minVal, maxVal, warns, err := getNumericThresholds(size, requestedMinValue, requestedMaxValue)
Expand All @@ -279,8 +292,14 @@ func validateNoiseNumericTypeAndSetLimit(
if warns.IsFatal() {
return nil, warns, nil
}
if requestedMinValue == nil {
requestedMinValue = &minVal
}
if requestedMaxValue == nil {
requestedMaxValue = &maxVal
}

limiter, err = transformers.NewNoiseNumericLimiter(minVal, maxVal)
limiter, err = transformers.NewNoiseNumericLimiter(*requestedMinValue, *requestedMaxValue)
if err != nil {
return nil, nil, fmt.Errorf("error creating limiter by size: %w", err)
}
Expand Down
13 changes: 13 additions & 0 deletions internal/db/postgres/transformers/noise_numeric_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,19 @@ func TestNoiseNumericTransformer_Transform(t *testing.T) {
input: "100",
result: result{min: 90, max: 110, regexp: `^-*\d+$`},
},
{
name: "numeric with thresholds",
columnName: "id_numeric",
params: map[string]toolkit.ParamsValue{
"min_ratio": toolkit.ParamsValue("0.2"),
"max_ratio": toolkit.ParamsValue("0.9"),
"min": toolkit.ParamsValue("0"),
"max": toolkit.ParamsValue("50"),
"decimal": toolkit.ParamsValue("4"),
},
input: "100",
result: result{min: 10, max: 190, regexp: `^-*\d+[.]*\d{0,4}$`},
},
}

for _, tt := range tests {
Expand Down
20 changes: 16 additions & 4 deletions internal/db/postgres/transformers/random_float.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,11 +138,23 @@ func NewFloatTransformer(ctx context.Context, driver *toolkit.Driver, parameters
}

if !dynamicMode {
if err := minParam.Scan(&minVal); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
minIsEmpty, err := minParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"min\" parameter: %w", err)
}
if !minIsEmpty {
if err = minParam.Scan(&minVal); err != nil {
return nil, nil, fmt.Errorf("error scanning \"min\" parameter: %w", err)
}
}
maxIsEmpty, err := maxParam.IsEmpty()
if err != nil {
return nil, nil, fmt.Errorf("error checking \"max\" parameter: %w", err)
}
if err := maxParam.Scan(&maxVal); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
if !maxIsEmpty {
if err = maxParam.Scan(&maxVal); err != nil {
return nil, nil, fmt.Errorf("error scanning \"max\" parameter: %w", err)
}
}
}

Expand Down
15 changes: 15 additions & 0 deletions internal/db/postgres/transformers/random_float_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,21 @@ func TestRandomFloatTransformer_Transform(t *testing.T) {
isNull: true,
},
},
{
name: "keep_null true and NULL seq",
columnName: "col_float8",
originalValue: "\\N",
params: map[string]toolkit.ParamsValue{
"min": toolkit.ParamsValue("0"),
"max": toolkit.ParamsValue("1000"),
"decimal": toolkit.ParamsValue("0"),
"keep_null": toolkit.ParamsValue("false"),
},
result: result{
min: 0,
max: 1000,
},
},
//{
// name: "text with default float8",
// params: map[string]toolkit.ParamsValue{
Expand Down
Loading

0 comments on commit ccb47c0

Please sign in to comment.