Fix tests

.github/workflows/test_mysql.yml

@@ -12,7 +12,7 @@ defaults:
 
 jobs:
 
-  masking:
+  generation_from_scratch:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -22,24 +22,96 @@ jobs:
           docker-compose pull
           docker-compose build
 
-      - name: Run the workflow
+      - name: Run databases
+        run: |
+          docker-compose run databases
+
+      - name: Verify input data
+        run: >
+          docker-compose run check scan -d input_db
+          -c /sodacl/configuration.yaml
+          /sodacl/checks_for_empty_input_db.yaml
+
+      - name: Run TDK
+        run: |
+          export CONFIG_FILE=config_generation_from_scratch.tdk.yaml
+          docker-compose run tdk
+
+      - name: Verify output data
+        run: >
+          docker-compose run check scan -d output_db
+          -c /sodacl/configuration.yaml
+          /sodacl/checks_for_generation_from_scratch.yaml
+
+
+  generation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build docker-compose
+        run: |
+          docker-compose pull
+          docker-compose build
+
+      - name: Run databases
         run: |
-          export CONFIG_FILE=config_masking.tdk.yaml
+          docker-compose -f docker-compose.yaml \
+            -f docker-compose-input-db.yaml run databases
+
+      - name: Verify input data
+        run: >
+          docker-compose run check scan -d input_db
+          -c /sodacl/configuration.yaml
+          /sodacl/checks_for_input_db.yaml
+
+      - name: Run TDK
+        run: |
+          export CONFIG_FILE=config_generation.tdk.yaml
           docker-compose -f docker-compose.yaml \
             -f docker-compose-input-db.yaml run tdk
 
+      - name: Verify output data
+        run: >
+          docker-compose run check scan -d output_db
+          -c /sodacl/configuration.yaml
+          /sodacl/checks_for_2x_generation.yaml
+
+
+  masking:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build docker-compose
+        run: |
+          docker-compose pull
+          docker-compose build
+
+      - name: Run databases
+        run: |
+          docker-compose -f docker-compose.yaml \
+            -f docker-compose-input-db.yaml run databases
+
       - name: Verify input data
         run: >
           docker-compose run check scan -d input_db
           -c /sodacl/configuration.yaml
           /sodacl/checks_for_input_db.yaml
 
+      - name: Run TDK
+        run: |
+          export CONFIG_FILE=config_masking.tdk.yaml
+          docker-compose -f docker-compose.yaml \
+            -f docker-compose-input-db.yaml run tdk
+
       - name: Verify output data
         run: >
           docker-compose run check scan -d output_db
           -c /sodacl/configuration.yaml
           /sodacl/checks_for_masking.yaml
 
+
   subsetting:
     runs-on: ubuntu-latest
     steps:
@@ -50,18 +122,23 @@ jobs:
           docker-compose pull
           docker-compose build
 
-      - name: Run the workflow
+      - name: Run databases
         run: |
-          export CONFIG_FILE=config_subsetting.tdk.yaml
           docker-compose -f docker-compose.yaml \
-            -f docker-compose-input-db.yaml run tdk
+            -f docker-compose-input-db.yaml run databases
 
       - name: Verify input data
         run: >
           docker-compose run check scan -d input_db
           -c /sodacl/configuration.yaml
           /sodacl/checks_for_input_db.yaml
 
+      - name: Run TDK
+        run: |
+          export CONFIG_FILE=config_subsetting.tdk.yaml
+          docker-compose -f docker-compose.yaml \
+            -f docker-compose-input-db.yaml run tdk
+
       - name: Verify output data
         run: >
           docker-compose run check scan -d output_db

mysql/README.md

@@ -2,28 +2,63 @@
 
 [Sakila](https://dev.mysql.com/doc/sakila/en) is a sample database schema that can be used in MySQL tutorials, articles, demos, etc.
 
+Clone this repo:
 ```shell
 git clone https://github.com/synthesized-io/pagila-tdk-demo
 cd pagila-tdk-demo/mysql
 ```
 
 
-## Masking of the existing data
+## Generation based on the empty schema
 
-![masking demo](masking.gif)
+![generation from scratch demo](generation_from_scratch.gif)
 
-Run TDK using Docker image:
+Run TDK:
 ```shell
-export CONFIG_FILE=config_masking.tdk.yaml
+export CONFIG_FILE=config_generation_from_scratch.tdk.yaml
+docker-compose run tdk
+```
+
+Check the ORIGINAL database with [`control sql-query`](control_query.sql):
+```shell
+usql mysql://root:admin@localhost:6000/sakila -f control_query.sql
+```
+
+Check the RESULTED database with [`control sql-query`](control_query.sql):
+```shell
+usql mysql://root:admin@localhost:6001/sakila -f control_query.sql
+```
+
+
+## Generation based on the existing data
+
+![generation demo](generation.gif)
+
+Run TDK:
+```shell
+export CONFIG_FILE=config_generation.tdk.yaml
 docker-compose -f docker-compose.yaml -f docker-compose-input-db.yaml run tdk
 ```
 
-Or run TDK CLI directly:
+Check the ORIGINAL database with [`control sql-query`](control_query.sql):
+```shell
+usql mysql://root:admin@localhost:6000/sakila -f control_query.sql
+```
+
+Check the RESULTED database with [`control sql-query`](control_query.sql):
+```shell
+usql mysql://root:admin@localhost:6001/sakila -f control_query.sql
+```
+
+
+## Masking of the existing data
+
+![masking demo](masking.gif)
+
+Run TDK:
 ```shell
-docker-compose -f docker-compose.yaml -f docker-compose-input-db.yaml run databases
-tdk \
-    --inventory-file inventory.yaml \
-    --config-file ./config_masking.tdk.yaml
+export CONFIG_FILE=config_masking.tdk.yaml
+docker-compose -f docker-compose.yaml -f docker-compose-input-db.yaml run tdk
 ```
 
 Check the ORIGINAL database with [`control sql-query`](control_query.sql):
@@ -47,14 +82,6 @@ export CONFIG_FILE=config_subsetting.tdk.yaml
 docker-compose -f docker-compose.yaml -f docker-compose-input-db.yaml run tdk
 ```
 
-Or run TDK CLI directly:
-```shell
-docker-compose -f docker-compose.yaml -f docker-compose-input-db.yaml run databases
-tdk \
-    --inventory-file inventory.yaml \
-    --config-file ./config_subsetting.tdk.yaml
-```
-
 Check the ORIGINAL database with [`control sql-query`](control_query.sql):
 ```shell
 usql mysql://root:admin@localhost:6000/sakila -f control_query.sql

mysql/config_generation.tdk.yaml

@@ -4,27 +4,22 @@ default_config:
 
 tables:
 
-  - table_name_with_schema: "public.film"
+  - table_name_with_schema: "sakila.film"
     transformations:
-      - columns: ["fulltext"]
-        mapping:
-          read: "?::text"
-          write: "?::tsvector"
-        params:
-          type: formatted_string_generator
-          pattern: "\\+44[0-9]{10}"
       - columns: ["rating"]
-        mapping:
-          read: "?::text"
-          write: "?::mpaa_rating"
         params:
           type: "categorical_generator"
           categories:
-            type: string
-            values: ["PG-13", "NC-17", "R", "PG", "G"]
-          probabilities: [0.223, 0.21, 0.195, 0.194, 0.178]
+            type: STRING
+            value_source: PROVIDED
+            values:
+              "PG-13": 0.223
+              "NC-17": 0.21
+              "R": 0.195
+              "PG": 0.194
+              "G": 0.178
 
-  - table_name_with_schema: "public.actor"
+  - table_name_with_schema: "sakila.actor"
     transformations:
       - columns: ["first_name", "last_name"]
         # An example how to preserve the uppercase
@@ -35,21 +30,21 @@ tables:
           type: person_generator
           column_templates: ["${first_name}", "${last_name}"]
 
-  - table_name_with_schema: "public.country"
+  - table_name_with_schema: "sakila.country"
     transformations:
       - columns: [ "country" ]
         params:
           type: address_generator
           column_templates: [ "${country}" ]
 
-  - table_name_with_schema: "public.city"
+  - table_name_with_schema: "sakila.city"
     transformations:
       - columns: [ "city" ]
         params:
           type: address_generator
           column_templates: [ "${city}" ]
 
-  - table_name_with_schema: "public.address"
+  - table_name_with_schema: "sakila.address"
     transformations:
       - columns: [ "address", "address2" ]
         params:
@@ -59,22 +54,33 @@ tables:
             - >
               ${country}, ${city}, ${street_name}, ${house_number},
               ${flat_number}, ${zip_code}
+          length_exceeded_mode: "TRUNCATE"
 
-  - table_name_with_schema: "public.staff"
+  - table_name_with_schema: "sakila.staff"
     transformations:
       - columns: ["first_name", "last_name", "email"]
         params:
           type: person_generator
           column_templates: ["${first_name}", "${last_name}", "${email}"]
 
-  - table_name_with_schema: "public.customer"
+  - table_name_with_schema: "sakila.customer"
     transformations:
       - columns: ["first_name", "last_name", "email"]
         params:
           type: person_generator
           column_templates: ["${first_name}", "${last_name}", "${email}"]
 
+  - table_name_with_schema: "sakila.rental"
+    transformations:
+      - columns: [ "rental_date" ]
+        params:
+          type: "date_generator"
+          mean: 2022-03-01T12:00:00Z
+          std: 7776000000
+          min: 2022-01-01T12:00:00Z
+          max: 2022-07-01T12:00:00Z
+
 table_truncation_mode: "TRUNCATE"
-schema_creation_mode: "DO_NOT_CREATE"
+schema_creation_mode: "DROP_AND_CREATE"
 cycle_resolution_strategy: "DELETE_NOT_REQUIRED"
 safety_mode: "RELAXED"