diff --git a/go.mod b/go.mod index 7d1a2225..450fde7d 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Masterminds/sprig/v3 v3.3.0 github.com/aws/aws-sdk-go v1.55.5 github.com/dchest/siphash v1.2.3 + github.com/expr-lang/expr v1.16.7 github.com/ggwhite/go-masker v1.1.0 github.com/go-faker/faker/v4 v4.5.0 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index b90501b2..393984a4 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= +github.com/expr-lang/expr v1.16.7 h1:gCIiHt5ODA0xIaDbD0DPKyZpM9Drph3b3lolYAYq2Kw= +github.com/expr-lang/expr v1.16.7/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= diff --git a/internal/db/postgres/context/table.go b/internal/db/postgres/context/table.go index 321f5260..808b810d 100644 --- a/internal/db/postgres/context/table.go +++ b/internal/db/postgres/context/table.go @@ -124,7 +124,7 @@ func validateAndBuildTablesConfig( // InitTransformation toolkit if len(tableCfg.Transformers) > 0 { for _, tc := range tableCfg.Transformers { - transformer, initWarnings, err := initTransformer(ctx, driver, tc, registry, types) + transformer, initWarnings, err := initTransformer(ctx, driver, tc, registry) if len(initWarnings) > 0 { for _, w := range initWarnings { // Enriching the tables context into meta diff --git a/internal/db/postgres/context/transformers.go b/internal/db/postgres/context/transformers.go index 0c611ad5..b32d53d0 100644 --- a/internal/db/postgres/context/transformers.go +++ b/internal/db/postgres/context/transformers.go @@ -27,8 +27,8 @@ func initTransformer( ctx context.Context, d *toolkit.Driver, c *domains.TransformerConfig, r *transformersUtils.TransformerRegistry, - types []*toolkit.Type, ) (*transformersUtils.TransformerContext, toolkit.ValidationWarnings, error) { + // TODO: Create var totalWarnings toolkit.ValidationWarnings td, ok := r.Get(c.Name) if !ok { @@ -42,7 +42,7 @@ func initTransformer( })) return nil, totalWarnings, nil } - transformer, warnings, err := td.Instance(ctx, d, c.Params, c.DynamicParams) + transformer, warnings, err := td.Instance(ctx, d, c.Params, c.DynamicParams, c.When) if err != nil { return nil, nil, fmt.Errorf("unable to init transformer: %w", err) } diff --git a/internal/db/postgres/dumpers/transformation_pipeline.go b/internal/db/postgres/dumpers/transformation_pipeline.go index f083253e..0d185018 100644 --- a/internal/db/postgres/dumpers/transformation_pipeline.go +++ b/internal/db/postgres/dumpers/transformation_pipeline.go @@ -132,7 +132,15 @@ func (tp *TransformationPipeline) Init(ctx context.Context) error { func (tp *TransformationPipeline) TransformSync(ctx context.Context, r *toolkit.Record) (*toolkit.Record, error) { var err error + var needTransform bool for _, t := range tp.table.TransformersContext { + needTransform, err = t.EvaluateWhen() + if err != nil { + return nil, NewDumpError(tp.table.Schema, tp.table.Name, tp.line, fmt.Errorf("error evaluating when condition: %w", err)) + } + if !needTransform { + continue + } _, err = t.Transformer.Transform(ctx, r) if err != nil { return nil, NewDumpError(tp.table.Schema, tp.table.Name, tp.line, err) diff --git a/internal/db/postgres/transformers/column_context.go b/internal/db/postgres/transformers/column_context.go index e522e69a..dbc6f455 100644 --- a/internal/db/postgres/transformers/column_context.go +++ b/internal/db/postgres/transformers/column_context.go @@ -49,7 +49,7 @@ func (cc *ColumnContext) GetValue() (any, error) { } func (cc *ColumnContext) GetRawValue() (any, error) { - return cc.rc.GetRawColumnValue(cc.columnName) + return cc.rc.GetColumnRawValue(cc.columnName) } func (cc *ColumnContext) GetColumnValue(name string) (any, error) { @@ -57,7 +57,7 @@ func (cc *ColumnContext) GetColumnValue(name string) (any, error) { } func (cc *ColumnContext) GetColumnRawValue(name string) (any, error) { - return cc.rc.GetRawColumnValue(name) + return cc.rc.GetColumnRawValue(name) } func (cc *ColumnContext) EncodeValue(v any) (any, error) { diff --git a/internal/db/postgres/transformers/dict_test.go b/internal/db/postgres/transformers/dict_test.go index 8eec7890..d1744248 100644 --- a/internal/db/postgres/transformers/dict_test.go +++ b/internal/db/postgres/transformers/dict_test.go @@ -41,6 +41,7 @@ func TestDictTransformer_Transform_with_fail(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -85,6 +86,7 @@ func TestDictTransformer_Transform_validation_error(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.NotEmpty(t, warnings) @@ -101,6 +103,7 @@ func TestDictTransformer_Transform_validation_error(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.NotEmpty(t, warnings) @@ -121,6 +124,7 @@ func TestDictTransformer_Transform_error_not_matched(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -149,6 +153,7 @@ func TestDictTransformer_Transform_use_default(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -180,6 +185,7 @@ func TestDictTransformer_Transform_with_int_values(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/email.go b/internal/db/postgres/transformers/email.go index 3ec79f3c..815dcb81 100644 --- a/internal/db/postgres/transformers/email.go +++ b/internal/db/postgres/transformers/email.go @@ -92,7 +92,7 @@ type EmailTransformer struct { originalDomain []byte randomBytesBuf []byte hexEncodedRandomBytesBuf []byte - rrctx *RoRecordContext + rctx *toolkit.RecordContext } func NewEmailTransformer(ctx context.Context, driver *toolkit.Driver, parameters map[string]toolkit.Parameterizer) (utils.Transformer, toolkit.ValidationWarnings, error) { @@ -142,7 +142,7 @@ func NewEmailTransformer(ctx context.Context, driver *toolkit.Driver, parameters return nil, nil, fmt.Errorf(`unable to scan "domain_part_template" param: %w`, err) } - rrctx := NewRoRecordContext() + rrctx := toolkit.NewRecordContext() funcMap := toolkit.FuncMap() if localPartTemplate != "" || domainTemplate != "" { for _, c := range driver.Table.Columns { @@ -208,8 +208,8 @@ func NewEmailTransformer(ctx context.Context, driver *toolkit.Driver, parameters domainTemplate: domainTmpl, validate: validate, buf: bytes.NewBuffer(nil), - hexEncodedRandomBytesBuf: make([]byte, hex.EncodedLen(maxLength)), - rrctx: rrctx, + hexEncodedRandomBytesBuf: make([]byte, hex.EncodedLen(emailTransformerGeneratorSize)), + rctx: rrctx, }, nil, nil } @@ -264,7 +264,7 @@ func (rit *EmailTransformer) setupTemplateContext(originalEmail []byte, r *toolk if rit.localPartTemplate == nil && rit.domainTemplate == nil && !rit.keepOriginalDomain { return nil } - rit.rrctx.setRecord(r) + rit.rctx.SetRecord(r) originalLocalPart, originalDomain, err := EmailParse(originalEmail) if err != nil { diff --git a/internal/db/postgres/transformers/hash_test.go b/internal/db/postgres/transformers/hash_test.go index 15ed8a71..f3c6151c 100644 --- a/internal/db/postgres/transformers/hash_test.go +++ b/internal/db/postgres/transformers/hash_test.go @@ -77,6 +77,7 @@ func TestHashTransformer_Transform_all_functions(t *testing.T) { context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -154,6 +155,7 @@ func TestHashTransformer_Transform_length_truncation(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -184,6 +186,7 @@ func TestHashTransformer_Transform_multiple_iterations(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/json_context.go b/internal/db/postgres/transformers/json_context.go index 2e36596f..f26862c3 100644 --- a/internal/db/postgres/transformers/json_context.go +++ b/internal/db/postgres/transformers/json_context.go @@ -47,7 +47,7 @@ func (jc *JsonContext) GetColumnValue(name string) (any, error) { } func (jc *JsonContext) GetColumnRawValue(name string) (any, error) { - return jc.rc.GetRawColumnValue(name) + return jc.rc.GetColumnRawValue(name) } func (jc *JsonContext) EncodeValueByColumn(name string, v any) (any, error) { diff --git a/internal/db/postgres/transformers/json_test.go b/internal/db/postgres/transformers/json_test.go index cef9bd33..7a23b10b 100644 --- a/internal/db/postgres/transformers/json_test.go +++ b/internal/db/postgres/transformers/json_test.go @@ -44,6 +44,7 @@ func TestJsonTransformer_Transform(t *testing.T) { ]`), }, nil, + "", ) require.NoError(t, err) assert.Empty(t, warnings) @@ -86,6 +87,7 @@ func TestJsonTransformer_Transform_with_template(t *testing.T) { "operations": opsData, }, nil, + "", ) require.NoError(t, err) assert.Empty(t, warnings) @@ -130,6 +132,7 @@ func TestJsonTransformer_Transform_null(t *testing.T) { "keep_null": toolkit.ParamsValue("false"), }, nil, + "", ) require.NoError(t, err) assert.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/masking_test.go b/internal/db/postgres/transformers/masking_test.go index 56b17b21..89d6673a 100644 --- a/internal/db/postgres/transformers/masking_test.go +++ b/internal/db/postgres/transformers/masking_test.go @@ -79,6 +79,7 @@ func TestMaskingTransformer_Transform(t *testing.T) { context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -109,6 +110,7 @@ func TestMaskingTransformer_type_validation(t *testing.T) { "type": toolkit.ParamsValue("unknown"), }, nil, + "", ) require.NoError(t, err) assert.Len(t, warnings, 1) diff --git a/internal/db/postgres/transformers/noise_date_test.go b/internal/db/postgres/transformers/noise_date_test.go index af84f361..35cd2ecc 100644 --- a/internal/db/postgres/transformers/noise_date_test.go +++ b/internal/db/postgres/transformers/noise_date_test.go @@ -118,6 +118,7 @@ func TestNoiseDateTransformer_Transform(t *testing.T) { context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/noise_float_test.go b/internal/db/postgres/transformers/noise_float_test.go index 9024d1a2..1c705c9a 100644 --- a/internal/db/postgres/transformers/noise_float_test.go +++ b/internal/db/postgres/transformers/noise_float_test.go @@ -131,6 +131,7 @@ func TestNoiseFloatTransformer_Transform(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/noise_int_test.go b/internal/db/postgres/transformers/noise_int_test.go index 012ffadd..0a6adc6b 100644 --- a/internal/db/postgres/transformers/noise_int_test.go +++ b/internal/db/postgres/transformers/noise_int_test.go @@ -104,6 +104,7 @@ func TestNoiseIntTransformer_Transform(t *testing.T) { context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/noise_numeric_test.go b/internal/db/postgres/transformers/noise_numeric_test.go index 61600277..5daf460f 100644 --- a/internal/db/postgres/transformers/noise_numeric_test.go +++ b/internal/db/postgres/transformers/noise_numeric_test.go @@ -121,6 +121,7 @@ func TestNoiseNumericTransformer_Transform(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_bool_test.go b/internal/db/postgres/transformers/random_bool_test.go index b76849fa..22463753 100644 --- a/internal/db/postgres/transformers/random_bool_test.go +++ b/internal/db/postgres/transformers/random_bool_test.go @@ -67,6 +67,7 @@ func TestRandomBoolTransformer_Transform(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_choice_test.go b/internal/db/postgres/transformers/random_choice_test.go index cc4bd721..77b9b5fb 100644 --- a/internal/db/postgres/transformers/random_choice_test.go +++ b/internal/db/postgres/transformers/random_choice_test.go @@ -26,6 +26,7 @@ func TestRandomChoiceTransformer_Transform_with_fail(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -57,6 +58,7 @@ func TestRandomChoiceTransformer_Transform_validation_error(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.NotEmpty(t, warnings) @@ -78,6 +80,7 @@ func TestRandomChoiceTransformer_Transform_json(t *testing.T) { context.Background(), driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_date_test.go b/internal/db/postgres/transformers/random_date_test.go index 91ef1297..5409d264 100644 --- a/internal/db/postgres/transformers/random_date_test.go +++ b/internal/db/postgres/transformers/random_date_test.go @@ -116,6 +116,7 @@ func TestTimestampTransformer_Transform(t *testing.T) { context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_float_test.go b/internal/db/postgres/transformers/random_float_test.go index dfd7cb7a..cc6e909a 100644 --- a/internal/db/postgres/transformers/random_float_test.go +++ b/internal/db/postgres/transformers/random_float_test.go @@ -159,6 +159,7 @@ func TestRandomFloatTransformer_Transform(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_int_test.go b/internal/db/postgres/transformers/random_int_test.go index e009c84e..5fe12ef3 100644 --- a/internal/db/postgres/transformers/random_int_test.go +++ b/internal/db/postgres/transformers/random_int_test.go @@ -131,6 +131,7 @@ func TestRandomIntTransformer_Transform_random_static(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -205,6 +206,7 @@ func TestRandomIntTransformer_Transform_random_dynamic(t *testing.T) { driver, tt.params, tt.dynamicParams, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -284,6 +286,7 @@ func TestRandomIntTransformer_Transform_deterministic_dynamic(t *testing.T) { driver, tt.params, tt.dynamicParams, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_ip_test.go b/internal/db/postgres/transformers/random_ip_test.go index 4ef4c89c..033c1327 100644 --- a/internal/db/postgres/transformers/random_ip_test.go +++ b/internal/db/postgres/transformers/random_ip_test.go @@ -69,6 +69,7 @@ func TestRandomIpTransformer_Transform_random_dynamic(t *testing.T) { driver, tt.params, tt.dynamicParams, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_mac_test.go b/internal/db/postgres/transformers/random_mac_test.go index 30dbae0d..a624c772 100644 --- a/internal/db/postgres/transformers/random_mac_test.go +++ b/internal/db/postgres/transformers/random_mac_test.go @@ -131,6 +131,7 @@ func TestRandomMacTransformer_Transform_random(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_numeric_test.go b/internal/db/postgres/transformers/random_numeric_test.go index 56d87d19..9f763bd6 100644 --- a/internal/db/postgres/transformers/random_numeric_test.go +++ b/internal/db/postgres/transformers/random_numeric_test.go @@ -95,6 +95,7 @@ func TestBigIntTransformer_Transform_random_static(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -175,6 +176,7 @@ func TestBigIntTransformer_Transform_random_dynamic(t *testing.T) { driver, tt.params, tt.dynamicParams, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -254,6 +256,7 @@ func TestBigIntTransformer_Transform_deterministic_dynamic(t *testing.T) { driver, tt.params, tt.dynamicParams, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_person_test.go b/internal/db/postgres/transformers/random_person_test.go index ac0b69a8..8165c107 100644 --- a/internal/db/postgres/transformers/random_person_test.go +++ b/internal/db/postgres/transformers/random_person_test.go @@ -32,6 +32,7 @@ func TestRandomPersonTransformer_Transform_static_fullname(t *testing.T) { driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -69,6 +70,7 @@ func TestRandomPersonTransformer_Transform_static_firstname(t *testing.T) { driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -106,6 +108,7 @@ func TestRandomPersonTransformer_Transform_static_lastname(t *testing.T) { driver, params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_string_test.go b/internal/db/postgres/transformers/random_string_test.go index 8db5fe04..1efd36da 100644 --- a/internal/db/postgres/transformers/random_string_test.go +++ b/internal/db/postgres/transformers/random_string_test.go @@ -91,6 +91,7 @@ func TestRandomStringTransformer_Transform(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_unix_timestamp_test.go b/internal/db/postgres/transformers/random_unix_timestamp_test.go index 83821476..44da8821 100644 --- a/internal/db/postgres/transformers/random_unix_timestamp_test.go +++ b/internal/db/postgres/transformers/random_unix_timestamp_test.go @@ -125,6 +125,7 @@ func TestUnixTimestampTransformer_Transform__positive_cases__static(t *testing.T context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -200,6 +201,7 @@ func TestUnixTimestampTransformer_Transform_null_cases(t *testing.T) { context.Background(), driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -276,6 +278,7 @@ func TestUnixTimestampTransformer_Transform_dynamic(t *testing.T) { driver, tt.params, tt.dynamicParams, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/random_uuid_test.go b/internal/db/postgres/transformers/random_uuid_test.go index 3a2358a5..4bf31f66 100644 --- a/internal/db/postgres/transformers/random_uuid_test.go +++ b/internal/db/postgres/transformers/random_uuid_test.go @@ -76,6 +76,7 @@ func TestUuidTransformer_Transform_uuid_type(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/real_address_test.go b/internal/db/postgres/transformers/real_address_test.go index 44a1dbff..d40e1372 100644 --- a/internal/db/postgres/transformers/real_address_test.go +++ b/internal/db/postgres/transformers/real_address_test.go @@ -44,6 +44,7 @@ func TestRealAddressTransformer_Transform(t *testing.T) { "columns": rawData, }, nil, + "", ) require.NoError(t, err) @@ -77,6 +78,7 @@ func TestMakeNewFakeTransformerFunction_parsing_error(t *testing.T) { "columns": rawData, }, nil, + "", ) require.Len(t, warnings, 1) require.Equal(t, "error parsing template", warnings[0].Msg) @@ -102,6 +104,7 @@ func TestMakeNewFakeTransformerFunction_validation_error(t *testing.T) { "columns": rawData, }, nil, + "", ) require.Len(t, warnings, 1) require.Equal(t, "error validating template", warnings[0].Msg) diff --git a/internal/db/postgres/transformers/regexp_replace_test.go b/internal/db/postgres/transformers/regexp_replace_test.go index a26f096c..fe27e797 100644 --- a/internal/db/postgres/transformers/regexp_replace_test.go +++ b/internal/db/postgres/transformers/regexp_replace_test.go @@ -52,6 +52,7 @@ func TestRegexpReplaceTransformer_Transform2(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/replace_test.go b/internal/db/postgres/transformers/replace_test.go index 996ec672..1d31675d 100644 --- a/internal/db/postgres/transformers/replace_test.go +++ b/internal/db/postgres/transformers/replace_test.go @@ -87,6 +87,7 @@ func TestReplaceTransformer_Transform(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -146,6 +147,7 @@ func TestReplaceTransformer_Transform_with_raw_value(t *testing.T) { driver, tt.params, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -185,6 +187,7 @@ func TestReplaceTransformer_Transform_with_validation_error(t *testing.T) { driver, params, nil, + "", ) require.NoError(t, err) assert.NotEmpty(t, warnings) diff --git a/internal/db/postgres/transformers/ro_record_context.go b/internal/db/postgres/transformers/ro_record_context.go deleted file mode 100644 index 671adac8..00000000 --- a/internal/db/postgres/transformers/ro_record_context.go +++ /dev/null @@ -1,47 +0,0 @@ -package transformers - -import ( - "github.com/greenmaskio/greenmask/pkg/toolkit" -) - -type RoRecordContext struct { - rc *toolkit.RecordContext -} - -func NewRoRecordContext() *RoRecordContext { - return &RoRecordContext{ - rc: &toolkit.RecordContext{}, - } -} - -func (cc *RoRecordContext) clean() { - cc.rc.Clean() -} - -func (cc *RoRecordContext) setRecord(r *toolkit.Record) { - cc.rc.SetRecord(r) -} - -func (cc *RoRecordContext) GetColumnValue(name string) (any, error) { - return cc.rc.GetColumnValue(name) -} - -func (cc *RoRecordContext) GetColumnRawValue(name string) (any, error) { - return cc.rc.GetRawColumnValue(name) -} - -func (cc *RoRecordContext) EncodeValueByColumn(name string, v any) (any, error) { - return cc.rc.EncodeValueByColumn(name, v) -} - -func (cc *RoRecordContext) DecodeValueByColumn(name string, v any) (any, error) { - return cc.rc.DecodeValueByColumn(name, v) -} - -func (cc *RoRecordContext) EncodeValueByType(name string, v any) (any, error) { - return cc.rc.EncodeValueByType(name, v) -} - -func (cc *RoRecordContext) DecodeValueByType(name string, v any) (any, error) { - return cc.rc.DecodeValueByType(name, v) -} diff --git a/internal/db/postgres/transformers/set_null_test.go b/internal/db/postgres/transformers/set_null_test.go index 7f10cdaf..dae225ae 100644 --- a/internal/db/postgres/transformers/set_null_test.go +++ b/internal/db/postgres/transformers/set_null_test.go @@ -37,6 +37,7 @@ func TestSetNullTransformer_Transform(t *testing.T) { "column": toolkit.ParamsValue(columnName), }, nil, + "", ) require.NoError(t, err) assert.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/template_record_test.go b/internal/db/postgres/transformers/template_record_test.go index 4a5424b6..e8b8866a 100644 --- a/internal/db/postgres/transformers/template_record_test.go +++ b/internal/db/postgres/transformers/template_record_test.go @@ -60,6 +60,7 @@ func TestTemplateRecordTransformer_Transform_date(t *testing.T) { "template": toolkit.ParamsValue(template), }, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) @@ -85,7 +86,7 @@ func TestTemplateRecordTransformer_Transform_date(t *testing.T) { func TestTemplateRecordTransformer_Transform_json(t *testing.T) { var columnName = "doc" var template = ` - {{ $val := .GetRawColumnValue "doc" }} + {{ $val := .GetColumnRawValue "doc" }} {{ jsonSet "name" "hello" $val | jsonValidate | .SetColumnValue "doc" }} ` @@ -109,6 +110,7 @@ func TestTemplateRecordTransformer_Transform_json(t *testing.T) { "template": toolkit.ParamsValue(template), }, nil, + "", ) require.NoError(t, err) require.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/template_test.go b/internal/db/postgres/transformers/template_test.go index 3da253cc..c1526288 100644 --- a/internal/db/postgres/transformers/template_test.go +++ b/internal/db/postgres/transformers/template_test.go @@ -48,6 +48,7 @@ func TestTemplateTransformer_Transform_int(t *testing.T) { "template": toolkit.ParamsValue(template), }, nil, + "", ) require.NoError(t, err) assert.Empty(t, warnings) @@ -86,6 +87,7 @@ func TestTemplateTransformer_Transform_timestamp(t *testing.T) { "template": toolkit.ParamsValue(template), }, nil, + "", ) require.NoError(t, err) assert.Empty(t, warnings) diff --git a/internal/db/postgres/transformers/utils/definition.go b/internal/db/postgres/transformers/utils/definition.go index c09ef43b..6db05e50 100644 --- a/internal/db/postgres/transformers/utils/definition.go +++ b/internal/db/postgres/transformers/utils/definition.go @@ -18,6 +18,9 @@ import ( "context" "fmt" + "github.com/expr-lang/expr" + "github.com/expr-lang/expr/vm" + "github.com/greenmaskio/greenmask/pkg/toolkit" ) @@ -56,10 +59,36 @@ type TransformerContext struct { Transformer Transformer StaticParameters map[string]*toolkit.StaticParameter DynamicParameters map[string]*toolkit.DynamicParameter + whenCond *vm.Program + whenEnv expr.Option + rc *toolkit.RecordContext +} + +func (tc *TransformerContext) SetRecord(r *toolkit.Record) { + tc.rc.SetRecord(r) +} + +func (tc *TransformerContext) EvaluateWhen() (bool, error) { + if tc.whenCond == nil { + return true, nil + } + + output, err := expr.Run(tc.whenCond, nil) + if err != nil { + return false, fmt.Errorf("unable to evaluate when condition: %w", err) + } + + cond, ok := output.(bool) + if ok { + return cond, nil + } + + return false, fmt.Errorf("when condition should return boolean, got (%T) and value %+v", cond, cond) } func (d *TransformerDefinition) Instance( ctx context.Context, driver *toolkit.Driver, rawParams map[string]toolkit.ParamsValue, dynamicParameters map[string]*toolkit.DynamicParamValue, + whenCond string, ) (*TransformerContext, toolkit.ValidationWarnings, error) { // DecodeValue parameters and get the pgcopy of parsed params, parametersWarnings, err := toolkit.InitParameters(driver, d.Parameters, rawParams, dynamicParameters) @@ -103,9 +132,51 @@ func (d *TransformerDefinition) Instance( res = append(res, schemaWarnings...) res = append(res, transformerWarnings...) + cond, rc, condWarns := compileCond(whenCond, driver) + res = append(res, condWarns...) + return &TransformerContext{ Transformer: t, StaticParameters: staticParams, DynamicParameters: dynamicParams, + whenCond: cond, + rc: rc, }, res, nil } + +func compileCond(whenCond string, driver *toolkit.Driver) (*vm.Program, *toolkit.RecordContext, toolkit.ValidationWarnings) { + if whenCond == "" { + return nil, nil, nil + } + rc, funcs := newRecordContext(driver) + + cond, err := expr.Compile(whenCond, funcs...) + if err != nil { + return nil, nil, toolkit.ValidationWarnings{ + toolkit.NewValidationWarning(). + SetSeverity(toolkit.ErrorValidationSeverity). + AddMeta("Error", err.Error()). + SetMsg("unable to compile when condition"), + } + } + + return cond, rc, nil +} + +func newRecordContext(driver *toolkit.Driver) (*toolkit.RecordContext, []expr.Option) { + var funcs []expr.Option + rctx := toolkit.NewRecordContext() + for _, c := range driver.Table.Columns { + + f := expr.Function( + c.Name, + func(name string) func(params ...any) (any, error) { + return func(params ...any) (any, error) { + return rctx.GetColumnRawValue(name) + } + }(c.Name), + ) + funcs = append(funcs, f) + } + return rctx, funcs +} diff --git a/internal/db/postgres/transformers/utils/definition_test.go b/internal/db/postgres/transformers/utils/definition_test.go index 02eb72e6..f23adfe5 100644 --- a/internal/db/postgres/transformers/utils/definition_test.go +++ b/internal/db/postgres/transformers/utils/definition_test.go @@ -100,7 +100,7 @@ func TestDefinition(t *testing.T) { "replace": []byte("2023-08-27 12:08:11.304895"), } - _, warnings, err := TestTransformerDefinition.Instance(context.Background(), driver, rawParams, nil) + _, warnings, err := TestTransformerDefinition.Instance(context.Background(), driver, rawParams, nil, "") require.NoError(t, err) assert.Empty(t, warnings) } diff --git a/internal/domains/config.go b/internal/domains/config.go index 84d9a5c3..7083e8b3 100644 --- a/internal/domains/config.go +++ b/internal/domains/config.go @@ -144,6 +144,7 @@ type TransformerConfig struct { // this is used only due to https://github.com/spf13/viper/issues/373 MetadataParams map[string]any `mapstructure:"-" yaml:"params,omitempty" json:"params,omitempty"` DynamicParams toolkit.DynamicParameters `mapstructure:"dynamic_params" yaml:"dynamic_params" json:"dynamic_params,omitempty"` + When string `mapstructure:"when" yaml:"when" json:"when,omitempty"` } type Table struct { @@ -154,6 +155,7 @@ type Table struct { Transformers []*TransformerConfig `mapstructure:"transformers" yaml:"transformers" json:"transformers,omitempty"` ColumnsTypeOverride map[string]string `mapstructure:"columns_type_override" yaml:"columns_type_override" json:"columns_type_override,omitempty"` SubsetConds []string `mapstructure:"subset_conds" yaml:"subset_conds" json:"subset_conds,omitempty"` + When string `mapstructure:"when" yaml:"when" json:"when,omitempty"` } // DummyConfig - This is a dummy config to the viper workaround diff --git a/pkg/toolkit/dynamic_parameter.go b/pkg/toolkit/dynamic_parameter.go index 10535f60..99bb2aff 100644 --- a/pkg/toolkit/dynamic_parameter.go +++ b/pkg/toolkit/dynamic_parameter.go @@ -58,7 +58,7 @@ func (dpc *DynamicParameterContext) GetValue() (any, error) { } func (dpc *DynamicParameterContext) GetRawValue() (any, error) { - return dpc.rc.GetRawColumnValue(dpc.column.Name) + return dpc.rc.GetColumnRawValue(dpc.column.Name) } func (dpc *DynamicParameterContext) GetColumnValue(name string) (any, error) { @@ -66,7 +66,7 @@ func (dpc *DynamicParameterContext) GetColumnValue(name string) (any, error) { } func (dpc *DynamicParameterContext) GetColumnRawValue(name string) (any, error) { - return dpc.rc.GetRawColumnValue(name) + return dpc.rc.GetColumnRawValue(name) } func (dpc *DynamicParameterContext) EncodeValue(v any) (any, error) { diff --git a/pkg/toolkit/template_record_context.go b/pkg/toolkit/template_record_context.go index e3a0da06..10daa7f7 100644 --- a/pkg/toolkit/template_record_context.go +++ b/pkg/toolkit/template_record_context.go @@ -61,7 +61,7 @@ func (rc *RecordContext) GetColumnValue(name string) (any, error) { return v.Value, nil } -func (rc *RecordContext) GetRawColumnValue(name string) (any, error) { +func (rc *RecordContext) GetColumnRawValue(name string) (any, error) { v, err := rc.record.GetRawColumnValueByName(name) if err != nil { return nil, err @@ -87,7 +87,7 @@ func (rc *RecordContext) SetColumnValue(name string, v any) (bool, error) { return true, nil } -func (rc *RecordContext) SetRawColumnValue(name string, v any) (bool, error) { +func (rc *RecordContext) SetColumnRawValue(name string, v any) (bool, error) { var val *RawValue switch vv := v.(type) { case NullType: