merge: Pulled from main

Makefile

@@ -20,5 +20,6 @@ coverage:
 install:
 	mv $(MAIN_PATH)/$(CMD_NAME) $(GOBIN)/$(CMD_NAME)
 
-build: $(CMD_FILES)
-	CGO_ENABLED=0 go build -ldflags="$(LDFLAGS)" -v -o $(CMD_NAME) $(MAIN_PATH)
+# The build flag -tags=viper_bind_struct has been added to avoid the need to bind each of the environment variables
+build: $(CMD_FILES)	
+	CGO_ENABLED=0 go build -tags=viper_bind_struct -ldflags="$(LDFLAGS)" -v -o $(CMD_NAME) $(MAIN_PATH)

cmd/greenmask/cmd/root.go

@@ -17,6 +17,7 @@ package cmd
 import (
 	"fmt"
 	"runtime/debug"
+	"strings"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/rs/zerolog"
@@ -99,10 +100,6 @@ func init() {
 	RootCmd.AddCommand(validate.Cmd)
 	RootCmd.AddCommand(show_transformer.Cmd)
 
-	if err := RootCmd.MarkPersistentFlagRequired("config"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
 	if err := viper.BindPFlag("log.format", RootCmd.PersistentFlags().Lookup("log-format")); err != nil {
 		log.Fatal().Err(err).Msg("")
 	}
@@ -111,13 +108,6 @@ func init() {
 		log.Fatal().Err(err).Msg("")
 	}
 
-	if err := viper.BindEnv("log.level", "LOG_LEVEL"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-	if err := viper.BindEnv("log.format", "LOG_FORMAT"); err != nil {
-		log.Fatal().Err(err).Msg("")
-	}
-
 	RootCmd.InitDefaultCompletionCmd()
 	RootCmd.InitDefaultHelpCmd()
 	RootCmd.InitDefaultVersionFlag()
@@ -136,16 +126,14 @@ func init() {
 func initConfig() {
 	if cfgFile != "" {
 		viper.SetConfigFile(cfgFile)
-	} else {
-		return
+		if err := viper.ReadInConfig(); err != nil {
+			log.Fatal().Err(err).Msg("error reading from config file")
+		}
 	}
 
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.AutomaticEnv()
 
-	if err := viper.ReadInConfig(); err != nil {
-		log.Fatal().Msgf("unable to read configUtils file: %s", err.Error())
-	}
-
 	decoderCfg := func(cfg *mapstructure.DecoderConfig) {
 		cfg.DecodeHook = mapstructure.ComposeDecodeHookFunc(
 			configUtils.ParamsToByteSliceHookFunc(),
@@ -157,5 +145,4 @@ func initConfig() {
 	if err := viper.Unmarshal(Config, decoderCfg); err != nil {
 		log.Fatal().Err(err).Msg("")
 	}
-
 }

docker/greenmask/Dockerfile

@@ -33,9 +33,18 @@ RUN apt-get update \
 
 COPY --from=build /var/lib/greenmask/greenmask /usr/bin
 
+RUN mkdir /home/greenmask \
+    && groupadd -g 10001 greenmask \
+    && useradd -u 10000 -g greenmask greenmask \
+    && chown -R greenmask:greenmask /home/greenmask
+
+USER greenmask:greenmask
+
 RUN mkdir ~/.bash_completions \
-    && greenmask completion bash > /root/.bash_completions/greenmask.bash \
-    && echo 'source /etc/bash_completion' >> /root/.bashrc \
-    && echo 'source /root/.bash_completions/greenmask.bash' >> /root/.bashrc
+    && greenmask completion bash > ~/.bash_completions/greenmask.bash \
+    && echo 'source /etc/bash_completion' >> ~/.bashrc \
+    && echo 'source ~/.bash_completions/greenmask.bash' >> ~/.bashrc
+
+WORKDIR /home/greenmask
 
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["greenmask"]

docs/configuration.md

@@ -1,4 +1,4 @@
-# Configuration
+    # Configuration
 
 The configuration is organized into six sections:
 
@@ -15,7 +15,7 @@ The configuration is organized into six sections:
 In the `common` section of the configuration, you can specify the following settings:
 
 * `pg_bin_path` — path to the PostgreSQL binaries. Note that the PostgreSQL server version must match the provided binaries.
-* `tmp_dir` — temporary directory for storing the table of contents files
+* `tmp_dir` — temporary directory for storing the table of contents files. Default value is `/tmp`
 
 !!! note
 
@@ -31,7 +31,7 @@ In the `log` section of the configuration, you can specify the following setting
 ## `storage` section
 
 In the `storage` section, you can configure the storage driver for storing the dumped data. Currently,
-two storage options are supported: `directory` and `s3`.
+two storage `type` options are supported: `directory` and `s3`.
 
 === "`directory` option"
 
@@ -40,7 +40,9 @@ two storage options are supported: `directory` and `s3`.
     Parameters include `path` which specifies the path to the directory in the filesystem where the dumps will be stored.
 
     ``` yaml title="directory storage config example"
-    directory:
+    storage:
+      type: "directory"
+      directory:
         path: "/home/user_name/storage_dir" # (1)
     ```
 
@@ -54,7 +56,7 @@ two storage options are supported: `directory` and `s3`.
     * `prefix` — a prefix for objects in the bucket, specified in path format
     * `region` — the S3 service region
     * `storage_class` — the storage class for performing object requests
-    * `disable_ssl` — disable SSL for interactions (default is `false`)
+    * `no_verify_ssl` — disable SSL certificate verification
     * `access_key_id` — access key for authentication
     * `secret_access_key` — secret access key for authentication
     * `session_token` — session token for authentication
@@ -69,12 +71,14 @@ two storage options are supported: `directory` and `s3`.
     * `use_accelerate` — enable S3 Accelerate feature
 
     ```yaml title="s3 storage config example for Minio running in Docker"
-    s3:
-      endpoint: "http://localhost:9000"
-      bucket: "testbucket"
-      region: "us-east-1"
-      access_key_id: "Q3AM3UQ867SPQQA43P2F"
-      secret_access_key: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
+    storage:  
+      type: "s3"
+      s3:
+        endpoint: "http://localhost:9000"
+        bucket: "testbucket"
+        region: "us-east-1"
+        access_key_id: "Q3AM3UQ867SPQQA43P2F"
+        secret_access_key: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
     ```
 
 ## `dump` section
@@ -293,3 +297,24 @@ scripts:
 2. **List of data stage scripts**. This section contains scripts that are executed before or after the restoration of the data section. The scripts include shell commands with parameters and SQL query files.
 3. **List of post-data stage scripts**. This section contains scripts that are executed before or after the restoration of the post-data section. The scripts include SQL queries and query files.
 4. **Command in the first argument and the parameters in the rest of the list**. When specifying a command to be executed in the scripts section, you provide the command name as the first item in a list, followed by any parameters or arguments for that command. The command and its parameters are provided as a list within the script configuration.
+
+## Environment variable configuration
+
+It's also possible to configure Greenmask through environment variables. 
+
+Greenmask will automatically parse any environment variable that matches the configuration in the config file by substituting the dot (`.`) separator for an underscore (`_`) and uppercasing it. As an example, the config file below would apply the same configuration as defining the `LOG_LEVEL=debug` environment variable
+
+```yaml title="config.yaml"
+log:
+  level: debug
+```
+
+### Postgres connection variables
+
+Additionaly, there are some environment variables exposed by the `dump` and `restore` commands to facilitate the connection configuration with a Postgres database
+
+* `PGHOST` - host used to connect to the postgres database
+* `PGPORT` - port where postgres is exposed
+* `PGDATABASE` - name of the database to dump/restore
+* `PGUSER` - username used to connect to the postgres database
+* `PGPASSWORD` - password used to authenticate to the postgres database

docs/mkdocs.yml

@@ -117,6 +117,9 @@ nav:
                   - Core custom functions: built_in_transformers/advanced_transformers/custom_functions/core_functions.md
                   - Faker function: built_in_transformers/advanced_transformers/custom_functions/faker_function.md
   - Release notes:
+      - Greenmask 0.1.10: release_notes/greenmask_0_1_10.md
+      - Greenmask 0.1.9: release_notes/greenmask_0_1_9.md
+      - Greenmask 0.1.8: release_notes/greenmask_0_1_8.md
       - Greenmask 0.1.7: release_notes/greenmask_0_1_7.md
       - Greenmask 0.1.6: release_notes/greenmask_0_1_6.md
       - Greenmask 0.1.5: release_notes/greenmask_0_1_5.md

docs/overrides/main.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
 
 {% block announce %}
-  Version 0.1.7 is <a href="https://github.com/GreenmaskIO/greenmask/releases/tag/v0.1.7">released</a>
+  Version 0.1.10 is <a href="https://github.com/GreenmaskIO/greenmask/releases/tag/v0.1.10">released</a>
 {% endblock %}

docs/release_notes/greenmask_0_1_10.md

@@ -0,0 +1,16 @@
+# Greenmask 0.1.10
+
+This release introduces improvements and bug fixes
+
+## Changes
+
+* Fixed panic caused in `RandomString` transformer
+* Fixed wrong table size calculation. Now the table size includes TOAST table size
+* Added custom transformer interaction API defaults if not set
+* Changed docker workdir to greenmask home
+* Removed bucket name from object path prefix
+
+## Assets
+
+To download the Greenmask binary compatible with your system, see
+the [release's assets list](https://github.com/GreenmaskIO/greenmask/releases/tag/v0.1.10).

docs/release_notes/greenmask_0_1_8.md

@@ -0,0 +1,20 @@
+# Greenmask 0.1.8
+
+This release introduces improvements and bug fixes
+
+## Improvements
+
+* Implemented `--exit-on-error` parameter for `pg_restore` run. But it does not play for "data" section restoration now. If any error is caused in `data` section greenmask exits with the error whether `--exit-on-error` was provided or not. This might be fixed later
+
+## Fixes
+
+* Fixed dependent objects dropping when running with the `restore` command with the `--clean` parameter. Useful when restoring and overriding only required tables
+* Fixed `show-dump` command output in text mode
+* Disabled CGO. Fixes problem when downloaded binary from repo cannot run
+* Fixed `delete` dump operation
+
+
+## Assets
+
+To download the Greenmask binary compatible with your system, see
+the [release's assets list](https://github.com/GreenmaskIO/greenmask/releases/tag/v0.1.8).

docs/release_notes/greenmask_0_1_9.md

@@ -0,0 +1,34 @@
+# Greenmask 0.1.9
+
+This release introduces improvements and bug fixes
+
+## Improvements
+
+* Implemented tables scoring according to the table size and transformation costs. This correctly spread the tables
+  dumping between the requested workers pool and reduces the execution time. Now greenmask introspects the table size,
+  adds the transformation scoring using the formula
+  `score = tableSizeInBytes + (tableSizeInBytes * 0.03 * tableTransformationsCount)`, and uses the strategy "Largest
+  First". The problem is described [here](https://github.com/GreenmaskIO/greenmask/discussions/50)
+* Introduced `no_verify_ssl` parameter for S3 storage
+* Adjusted Dockerfile
+    * Changed entrypoint to `greenmask` binary
+    * The `greenmask` container now runs under `greenmask` user and groups
+* Refactored storage config structure. Now it contains the `type` that is used for the storage type determination
+* Most of the attributes may be overridden with environment variables where the letters are capitalized and the dots
+  are replaced with underscores. For instance, the setting `storage.type` might be represented with the environment
+  variable `STORAGE_TYPE`
+* Parameter `--config` is not required anymore. This simplifies the greenmask utility user experience
+* Directory storage set as the default
+* Set the default temporary directory as `/tmp`
+* Added environment variable section to the configuration docs
+
+## Fixes
+
+* Fixed `S3_REGION` environment variable usage. Tested cases where the S3 storage is set up using `S3` variables that
+  uses by `github.com/aws/aws-sdk-go`
+* Updated project dependencies to the latest version
+
+## Assets
+
+To download the Greenmask binary compatible with your system, see
+the [release's assets list](https://github.com/GreenmaskIO/greenmask/releases/tag/v0.1.9).

go.mod

@@ -4,33 +4,35 @@ go 1.21
 
 require (
 	github.com/Masterminds/sprig/v3 v3.2.3
-	github.com/aws/aws-sdk-go v1.49.14
+	github.com/aws/aws-sdk-go v1.51.14
+	github.com/dchest/siphash v1.2.3
 	github.com/ggwhite/go-masker v1.1.0
-	github.com/go-faker/faker/v4 v4.2.0
-	github.com/google/uuid v1.5.0
+	github.com/go-faker/faker/v4 v4.4.0
+	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx-shopspring-decimal v0.0.0-20220624020537-1d36b5a1853e
-	github.com/jackc/pgx/v5 v5.5.1
+	github.com/jackc/pgx/v5 v5.5.5
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.31.0
+	github.com/rs/zerolog v1.32.0
+	github.com/shopspring/decimal v1.3.1
 	github.com/spaolacci/murmur3 v1.1.0
 	github.com/spf13/cast v1.6.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/viper v1.18.2
-	github.com/stretchr/testify v1.8.4
-	github.com/tidwall/gjson v1.17.0
+	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tidwall/sjson v1.2.5
-	golang.org/x/sync v0.5.0
+	golang.org/x/crypto v0.21.0
+	golang.org/x/sync v0.6.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/dchest/siphash v1.2.3 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
@@ -49,7 +51,6 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -58,9 +59,8 @@ require (
 	github.com/tidwall/pretty v1.2.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )