V0.2

cmd/greenmask/cmd/list_transformers/list_transformers.go

@@ -155,7 +155,7 @@ func listTransformersText(registry *utils.TransformerRegistry) error {
 	return nil
 }
 
-func getColumnTypes(p *toolkit.Parameter) []string {
+func getColumnTypes(p *toolkit.ParameterDefinition) []string {
 	if p.ColumnProperties != nil && len(p.ColumnProperties.AllowedTypes) > 0 {
 		return p.ColumnProperties.AllowedTypes
 	}

cmd/greenmask/cmd/show_transformer/show_transformer.go

@@ -82,7 +82,7 @@ func run(name string) error {
 }
 
 func showTransformerJson(registry *utils.TransformerRegistry, transformerName string) error {
-	var transformers []*utils.Definition
+	var transformers []*utils.TransformerDefinition
 
 	def, ok := registry.M[transformerName]
 	if ok {
@@ -114,8 +114,8 @@ func showTransformerText(registry *utils.TransformerRegistry, name string) error
 		if p.DefaultValue != nil {
 			data = append(data, []string{def.Properties.Name, "parameters", p.Name, "default", string(p.DefaultValue), ""})
 		}
-		if p.LinkParameter != "" {
-			data = append(data, []string{def.Properties.Name, "parameters", p.Name, "linked_parameter", p.LinkParameter, ""})
+		if p.LinkColumnParameter != "" {
+			data = append(data, []string{def.Properties.Name, "parameters", p.Name, "linked_parameter", p.LinkColumnParameter, ""})
 		}
 		if p.CastDbType != "" {
 			data = append(data, []string{def.Properties.Name, "parameters", p.Name, "cast_to_db_type", p.CastDbType, ""})
@@ -146,7 +146,7 @@ func showTransformerText(registry *utils.TransformerRegistry, name string) error
 	return nil
 }
 
-func getTransformerDefinition(registry *utils.TransformerRegistry, name string) (*utils.Definition, error) {
+func getTransformerDefinition(registry *utils.TransformerRegistry, name string) (*utils.TransformerDefinition, error) {
 	def, ok := registry.M[name]
 	if ok {
 		return def, nil

go.mod

@@ -5,6 +5,7 @@ go 1.21
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/aws/aws-sdk-go v1.51.14
+	github.com/dchest/siphash v1.2.3
 	github.com/ggwhite/go-masker v1.1.0
 	github.com/go-faker/faker/v4 v4.4.0
 	github.com/google/uuid v1.6.0
@@ -15,6 +16,8 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.32.0
+	github.com/shopspring/decimal v1.3.1
+	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.6.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
@@ -48,7 +51,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

go.sum

@@ -12,6 +12,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
+github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
 github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
@@ -93,6 +95,8 @@ github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5g
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
 github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
+github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
 github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=

internal/db/postgres/cmd/validate.go

@@ -273,7 +273,7 @@ func (v *Validate) dumpTables(ctx context.Context) error {
 	var tablesWithTransformers []entries.Entry
 	for _, item := range v.context.DataSectionObjects {
 
-		if t, ok := item.(*entries.Table); ok && len(t.Transformers) > 0 {
+		if t, ok := item.(*entries.Table); ok && len(t.TransformersContext) > 0 {
 			t.ValidateLimitedRecords = v.config.Validate.RowsLimit
 			tablesWithTransformers = append(tablesWithTransformers, t)
 		}

internal/db/postgres/cmd/validate_utils/json_document_test.go

@@ -80,7 +80,7 @@ func TestJsonDocument_GetRecords(t *testing.T) {
 	//	panic(err.Error())
 	//}
 	//row := pgcopy.NewRow(1)
-	//_ = row.Decode([]byte(value))
+	//_ = row.DecodeValue([]byte(value))
 	//r := toolkit.NewRecord(
 	//	driver,
 	//)
@@ -176,8 +176,10 @@ func getTableAndRows() (table *entries.Table, original, transformed [][]byte) {
 	}
 
 	table = &entries.Table{
-		Table:        t,
-		Transformers: []utils.Transformer{&testTransformer{}},
+		Table: t,
+		TransformersContext: []*utils.TransformerContext{
+			{Transformer: &testTransformer{}},
+		},
 	}
 
 	return table, original, transformed

internal/db/postgres/cmd/validate_utils/utils.go

@@ -13,8 +13,8 @@ const nullStringValue = "NULL"
 
 func getAffectedColumns(t *entries.Table) map[string]struct{} {
 	affectedColumns := make(map[string]struct{})
-	for _, tr := range t.Transformers {
-		ac := tr.GetAffectedColumns()
+	for _, tr := range t.TransformersContext {
+		ac := tr.Transformer.GetAffectedColumns()
 		for _, name := range ac {
 			affectedColumns[name] = struct{}{}
 		}

internal/db/postgres/context/context.go

@@ -16,7 +16,9 @@ package context
 
 import (
 	"context"
+	"encoding/hex"
 	"fmt"
+	"os"
 	"slices"
 
 	"github.com/jackc/pgx/v5"
@@ -57,6 +59,17 @@ func NewRuntimeContext(
 	ctx context.Context, tx pgx.Tx, cfg []*domains.Table, r *transformersUtils.TransformerRegistry, opt *pgdump.Options,
 	version int,
 ) (*RuntimeContext, error) {
+	var salt []byte
+	saltHex := os.Getenv("GREENMASK_GLOBAL_SALT")
+	if saltHex != "" {
+		salt = make([]byte, hex.DecodedLen(len(saltHex)))
+		_, err := hex.Decode(salt, []byte(saltHex))
+		if err != nil {
+			return nil, fmt.Errorf("error decoding salt from hex: %w", err)
+		}
+	}
+	ctx = context.WithValue(ctx, "salt", salt)
+
 	typeMap := tx.Conn().TypeMap()
 	types, err := getCustomTypesUsedInTables(ctx, tx)
 	if err != nil {

internal/db/postgres/context/queries.go

@@ -73,12 +73,14 @@ var (
 		    a.atttypid::TEXT::INT                          	as typeoid,
 		  	pg_catalog.format_type(a.atttypid, a.atttypmod) as typename,
 		  	a.attnotnull 									as notnull,
-		  	a.atttypmod 									as mod,
-		  	a.attnum 										as num
+		  	a.atttypmod 									as att_len,
+		  	a.attnum 										as num,
+		  	t.typlen 										as type_len
 			{{ if ge .Version 120000 }}
 		  	,a.attgenerated != ''	    				    as attgenerated
 			{{ end }}
 		FROM pg_catalog.pg_attribute a
+			JOIN pg_catalog.pg_type t ON a.atttypid = t.oid
 		WHERE a.attrelid = $1 AND a.attnum > 0 AND NOT a.attisdropped
 		ORDER BY a.attnum
 	`))
@@ -116,7 +118,8 @@ var (
 			   GROUP BY ct.oid
 			)
 			SELECT pt.oid::TEXT::INT         AS oid,
-				   twc.chain::INT[]          AS chain,
+				   twc.chain::INT[]          AS chain_oids,
+                   (select array_agg(t.typname)::TEXT[] FROM pg_type t WHERE t.oid = ANY (twc.chain)) AS chain_names,
 				   pn.nspname                AS schema,
 				   pt.typname                AS name,
 				   pt.typlen                 AS len,

internal/db/postgres/context/table.go

@@ -58,19 +58,37 @@ func validateAndBuildTablesConfig(
 			}
 			table.Constraints = constraints
 
+			// Assigning overridden column types for driver initialization
+			if tableCfg.ColumnsTypeOverride != nil {
+				for _, c := range table.Columns {
+					overridingType, ok := tableCfg.ColumnsTypeOverride[c.Name]
+					if ok {
+						c.OverriddenTypeName = overridingType
+					}
+				}
+			}
+
 			// Assign columns and transformersMap if were found
 			columns, err := getColumnsConfig(ctx, tx, table.Oid, version)
 			if err != nil {
 				return nil, nil, err
 			}
 			table.Columns = columns
 
-			driver, err := toolkit.NewDriver(table.Table, types, tableCfg.ColumnsTypeOverride)
+			driver, driverWarnings, err := toolkit.NewDriver(table.Table, types)
 			if err != nil {
 				return nil, nil, fmt.Errorf("unnable to initialise driver: %w", err)
 			}
 			table.Driver = driver
 
+			if len(driverWarnings) > 0 {
+				for _, w := range driverWarnings {
+					w.AddMeta("SchemaName", table.Schema).
+						AddMeta("TableName", table.Name)
+				}
+				warnings = append(warnings, driverWarnings...)
+			}
+
 			// InitTransformation toolkit
 			if len(tableCfg.Transformers) > 0 {
 				for _, tc := range tableCfg.Transformers {
@@ -84,11 +102,13 @@ func validateAndBuildTablesConfig(
 
 						}
 					}
+					// Not only errors might be in driver initialization but also a warnings that's why we have to add
+					// append medata to validation warnings and the check error and return error with warnings
 					if err != nil {
 						return nil, warnings, err
 					}
 					warnings = append(warnings, initWarnings...)
-					table.Transformers = append(table.Transformers, transformer)
+					table.TransformersContext = append(table.TransformersContext, transformer)
 				}
 			}
 
@@ -178,6 +198,7 @@ func getTable(ctx context.Context, tx pgx.Tx, t *domains.Table) ([]*entries.Tabl
 }
 
 func getColumnsConfig(ctx context.Context, tx pgx.Tx, oid toolkit.Oid, version int) ([]*toolkit.Column, error) {
+	defaultTypeMap := pgtype.NewMap()
 	var res []*toolkit.Column
 	buf := bytes.NewBuffer(nil)
 	err := TableColumnsQuery.Execute(
@@ -198,14 +219,21 @@ func getColumnsConfig(ctx context.Context, tx pgx.Tx, oid toolkit.Oid, version i
 		column := toolkit.Column{Idx: idx}
 		if version >= 120000 {
 			err = rows.Scan(&column.Name, &column.TypeOid, &column.TypeName,
-				&column.NotNull, &column.Length, &column.Num, &column.IsGenerated)
+				&column.NotNull, &column.Length, &column.Num, &column.TypeLength, &column.IsGenerated)
 		} else {
 			err = rows.Scan(&column.Name, &column.TypeOid, &column.TypeName,
 				&column.NotNull, &column.Length, &column.Num)
 		}
 		if err != nil {
 			return nil, fmt.Errorf("cannot scan tableColumnQuery: %w", err)
 		}
+		column.CanonicalTypeName = column.TypeName
+		// Getting canonical type name if exists. For instance - PostgreSQL type Integer is alias for int4
+		// (int4 - canonical type name)
+		canonicalType, ok := defaultTypeMap.TypeForOID(uint32(column.TypeOid))
+		if ok {
+			column.CanonicalTypeName = canonicalType.Name
+		}
 		res = append(res, &column)
 		idx++
 	}

internal/db/postgres/context/transformers.go

@@ -28,7 +28,7 @@ func initTransformer(
 	c *domains.TransformerConfig,
 	r *transformersUtils.TransformerRegistry,
 	types []*toolkit.Type,
-) (transformersUtils.Transformer, toolkit.ValidationWarnings, error) {
+) (*transformersUtils.TransformerContext, toolkit.ValidationWarnings, error) {
 	var totalWarnings toolkit.ValidationWarnings
 	td, ok := r.Get(c.Name)
 	if !ok {
@@ -42,7 +42,7 @@ func initTransformer(
 			}))
 		return nil, totalWarnings, nil
 	}
-	transformer, warnings, err := td.Instance(ctx, d, c.Params, types)
+	transformer, warnings, err := td.Instance(ctx, d, c.Params, c.DynamicParams)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to init transformer: %w", err)
 	}

internal/db/postgres/context/types.go

@@ -36,7 +36,7 @@ func getCustomTypesUsedInTables(ctx context.Context, tx pgx.Tx) ([]*toolkit.Type
 	for rows.Next() {
 		t := &toolkit.Type{}
 		var hasDomainConstraint bool
-		err = rows.Scan(&t.Oid, &t.Chain, &t.Schema, &t.Name, &t.Length, &t.Kind,
+		err = rows.Scan(&t.Oid, &t.ChainOids, &t.ChainNames, &t.Schema, &t.Name, &t.Length, &t.Kind,
 			&t.ComposedRelation, &t.ElementType, &t.ArrayType, &t.NotNull, &t.BaseType,
 			&hasDomainConstraint)
 		if err != nil {
@@ -45,8 +45,9 @@ func getCustomTypesUsedInTables(ctx context.Context, tx pgx.Tx) ([]*toolkit.Type
 		if hasDomainConstraint {
 			domainsWithConstraint = append(domainsWithConstraint, t)
 		}
-		if t.Kind == 'd' && len(t.Chain) > 0 {
-			t.RootBuiltInType = t.Chain[len(t.Chain)-1]
+		if t.Kind == 'd' && len(t.ChainOids) > 0 {
+			t.RootBuiltInTypeOid = t.ChainOids[len(t.ChainOids)-1]
+			t.RootBuiltInTypeName = t.ChainNames[len(t.ChainNames)-1]
 		}
 		_, exists := tx.Conn().TypeMap().TypeForOID(uint32(t.Oid))
 		if !exists {

internal/db/postgres/dumpers/table.go

@@ -69,7 +69,7 @@ func (td *TableDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Stora
 		func() error {
 			var pipeline Pipeliner
 			var err error
-			if len(td.table.Transformers) > 0 {
+			if len(td.table.TransformersContext) > 0 {
 				if td.validate {
 					pipeline, err = NewValidationPipeline(gtx, eg, td.table, w)
 					if err != nil {