synthesized-io · mgramin · Feb 20, 2024 · Feb 20, 2024 · Feb 20, 2024 · Feb 20, 2024
diff --git a/db2/config_masking.tdk.dot b/db2/config_masking.tdk.dot
@@ -0,0 +1,41 @@
+strict digraph G {
+  1 [ label="InputDatabase [id=default_source]" shape="folder" ];
+  2 [ label="OutputDatabase [id=default_target]" shape="folder" ];
+  3 [ label="TableMasking [PADT_ORS.C_PADT_ACCOUNT]" shape="ellipse" ];
+  4 [ label="InputTableRead [PADT_ORS.C_PADT_ACCOUNT]" shape="cylinder" ];
+  5 [ label="OutputTableRead [PADT_ORS.C_PADT_ACCOUNT]" shape="cylinder" ];
+  6 [ label="TableStore [PADT_ORS.C_PADT_ACCOUNT]" shape="cylinder" ];
+  7 [ label="TableMasking [PADT_ORS.C_PADT_PARTY]" shape="ellipse" ];
+  8 [ label="InputTableRead [PADT_ORS.C_PADT_PARTY]" shape="cylinder" ];
+  9 [ label="OutputTableRead [PADT_ORS.C_PADT_PARTY]" shape="cylinder" ];
+  10 [ label="TableStore [PADT_ORS.C_PADT_PARTY]" shape="cylinder" ];
+  11 [ label="TableMasking [PADT_ORS.C_PTY_TO_ACCT]" shape="ellipse" ];
+  12 [ label="InputTableRead [PADT_ORS.C_PTY_TO_ACCT]" shape="cylinder" ];
+  13 [ label="TableStore [PADT_ORS.C_PTY_TO_ACCT]" shape="cylinder" ];
+  14 [ label="InputAlignment [PADT_ORS.C_PTY_TO_ACCT]" shape="ellipse" ];
+  15 [ label="TableFilter [streamed=PADT_ORS.C_PTY_TO_ACCT build=PADT_ORS.C_PADT_PARTY,build=PADT_ORS.C_PADT_ACCOUNT]" shape="ellipse" ];
+  16 [ label="TableSample [PADT_ORS.C_PTY_TO_ACCT]" shape="ellipse" ];
+  17 [ label="TableSample [PADT_ORS.C_PADT_ACCOUNT]" shape="ellipse" ];
+  18 [ label="TableSample [PADT_ORS.C_PADT_PARTY]" shape="ellipse" ];
+  1 -> 4 [ label="belongs" style="dotted" arrowhead="crow" ];
+  2 -> 6 [ label="belongs" style="dotted" arrowhead="crow" ];
+  1 -> 8 [ label="belongs" style="dotted" arrowhead="crow" ];
+  2 -> 10 [ label="belongs" style="dotted" arrowhead="crow" ];
+  1 -> 12 [ label="belongs" style="dotted" arrowhead="crow" ];
+  2 -> 13 [ label="belongs" style="dotted" arrowhead="crow" ];
+  12 -> 14 [ label="data_flow" style="solid" arrowhead="normal" ];
+  15 -> 16 [ label="data_flow" style="solid" arrowhead="normal" ];
+  10 -> 9 [ label="await" style="solid" arrowhead="normal" ];
+  9 -> 15 [ label="table_reference" style="solid" arrowhead="normal" ];
+  14 -> 11 [ label="table_reference" style="solid" arrowhead="normal" ];
+  6 -> 5 [ label="await" style="solid" arrowhead="normal" ];
+  5 -> 15 [ label="table_reference" style="solid" arrowhead="normal" ];
+  11 -> 15 [ label="data_flow" style="solid" arrowhead="normal" ];
+  16 -> 13 [ label="data_flow" style="solid" arrowhead="normal" ];
+  3 -> 17 [ label="data_flow" style="solid" arrowhead="normal" ];
+  17 -> 6 [ label="data_flow" style="solid" arrowhead="normal" ];
+  4 -> 3 [ label="data_flow" style="solid" arrowhead="normal" ];
+  7 -> 18 [ label="data_flow" style="solid" arrowhead="normal" ];
+  18 -> 10 [ label="data_flow" style="solid" arrowhead="normal" ];
+  8 -> 7 [ label="data_flow" style="solid" arrowhead="normal" ];
+}
diff --git a/db2/config_masking.tdk.yaml b/db2/config_masking.tdk.yaml
@@ -1,8 +1,50 @@
 default_config:
-  mode: MASKING
+  mode: KEEP
+  use_working_directory: true
+  # mode: GENERATION
+  # target_row_number: 5_000_000
+  target_ratio: 0.0
+  insert_batch_size: 50000
 
 
-table_truncation_mode: "TRUNCATE"
-schema_creation_mode: DROP_AND_CREATE
-cycle_resolution_strategy: "DEFER_FOREIGN_KEY"
+tables:
+  - table_name_with_schema: "PADT_ORS.C_PADT_ACCOUNT"
+    target_ratio: 0.01
+  #   filter: "ROWID_OBJECT BETWEEN '0' AND '111I6'"
+
+  - table_name_with_schema: "PADT_ORS.C_PADT_PARTY"
+    target_ratio: 0.01
+  #   filter: "ROWID_OBJECT IN (SELECT act.ROWID_PARTY FROM PADT_ORS.C_PTY_TO_ACCT act WHERE act.ROWID_ACCT BETWEEN '0' AND '111I6')"
+
+  - table_name_with_schema: "PADT_ORS.C_PTY_TO_ACCT"
+    target_ratio: 0.01
+
+table_truncation_mode: DO_NOT_TRUNCATE
+schema_creation_mode: DO_NOT_CREATE
 safety_mode: "RELAXED"
+schemas:
+  - "PADT_ORS"
+
+
+
+metadata:
+  tables:
+  - table_name_with_schema: "PADT_ORS.C_PTY_TO_ACCT"
+    foreign_keys:
+     fk_party:
+      referred_schema: "PADT_ORS"
+      referred_table: "C_PADT_PARTY"
+      columns:
+      - column: "ROWID_PARTY"
+        referred_column: "ROWID_OBJECT"
+     fk_account:
+      referred_schema: "PADT_ORS"
+      referred_table: "C_PADT_ACCOUNT"
+      columns:
+      - column: "ROWID_ACCT"
+        referred_column: "ROWID_OBJECT"
+
+
+
+
+
diff --git a/db2/docker-compose.yaml b/db2/docker-compose.yaml
@@ -36,32 +36,40 @@ services:
       start_period: 10s
 
 
+  databases:
+    container_name: databases
+    image: docker/whalesay
+    command: >
+      cowsay
+        "Both databases have been started and are ready for TDK exercises!"
+    depends_on:
+      input_db:
+        condition: service_healthy
+      output_db:
+        condition: service_healthy
+
+
   tdk:
     extends:
       file: ../parent-compose.yml
       service: tdk
     environment:
-      SYNTHESIZED_INPUT_URL: jdbc:db2://input_db:50000/SAMPLE
+      SYNTHESIZED_INPUT_URL: jdbc:db2://output_db:50000/testdb
       SYNTHESIZED_INPUT_USERNAME: DB2INST1
       SYNTHESIZED_INPUT_PASSWORD: test
-      SYNTHESIZED_OUTPUT_URL: jdbc:db2://output_db:50000/testdb
+      SYNTHESIZED_OUTPUT_URL: jdbc:db2://input_db:50000/SAMPLE
       SYNTHESIZED_OUTPUT_USERNAME: DB2INST1
       SYNTHESIZED_OUTPUT_PASSWORD: test
       SYNTHESIZED_USERCONFIG_FILE: /app/config.yaml
       TDK_WORKINGDIRECTORY_PATH: /app/data
       TDK_WORKINGDIRECTORY_ENABLED: "true"
       SYNTHESIZED_KEY: ${SYNTHESIZED_KEY}
       JAVA_TOOL_OPTIONS: >
-        -Dlogging.level.io.synthesized.tdk.executor.lite.LiteTransformer=INFO
-        -Dlogging.level.io.synthesized.tdk=WARN
-        -Dlogging.level.com.zaxxer.hikari=WARN
-        -Dlogging.level.org.reflections=WARN
-        -Dlogging.level.org.jooq=WARN
         -XX:+UseContainerSupport
         -XX:MaxRAMPercentage=80.0
         -Dspring.main.banner-mode=CONSOLE
         -Dspring.banner.location=file:/app/banner.txt
     volumes:
       - ./${CONFIG_FILE:-config_masking.tdk.yaml}:/app/config.yaml
       - ../banner.txt:/app/banner.txt
-      - ../logback-lite-executor.xml:/app/logback-lite-executor.xml
+      # - ../logback-lite-executor.xml:/app/logback-lite-executor.xml
diff --git a/mssqlserver/config.tdk.yaml b/mssqlserver/config.tdk.yaml
@@ -0,0 +1,27 @@
+default_config:
+  # mode: MASKING
+  mode: GENERATION
+  # target_row_number: 10
+  target_ratio: 2
+
+
+# tables:
+#   - table_name_with_schema: "dbo.test_date_types_table"
+#     transformations:
+#       - columns:
+#         - "datetime_column"
+#         params:
+#           type: "date_time_unique_hashing"
+#           min: "1926-04-08T03:17:17.786Z"
+#           max: "2054-03-30T16:54:42.16Z"
+#       - columns:
+#         - "datetime2_column"
+#         params:
+#           type: "date_time_unique_hashing"
+#           min: "1926-04-08T03:17:17.786Z"
+#           max: "2054-03-30T16:54:42.16Z"
+
+
+table_truncation_mode: "TRUNCATE"
+schema_creation_mode: DO_NOT_CREATE
+safety_mode: "RELAXED"
diff --git a/mssqlserver/docker-compose.yaml b/mssqlserver/docker-compose.yaml
@@ -0,0 +1,62 @@
+version: '3'
+
+services:
+
+  input_db:
+    container_name: input_db
+    image: mcr.microsoft.com/mssql/server:2017-latest
+    ports:
+      - 6000:1433
+    environment:
+      MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Secret_password_1}
+      ACCEPT_EULA: Y
+    healthcheck:
+      test: >
+        echo 'select 1;' | /opt/mssql-tools/bin/sqlcmd -S 127.0.0.1 -U sa -P Secret_password_1
+      interval: 3s
+      timeout: 3s
+      retries: 10
+
+
+  output_db:
+    container_name: output_db
+    image: mcr.microsoft.com/mssql/server:2017-latest
+    ports:
+      - 6001:1433
+    environment:
+      MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Secret_password_1}
+      ACCEPT_EULA: Y
+    healthcheck:
+      test: >
+        echo 'select 1;' | /opt/mssql-tools/bin/sqlcmd -S 127.0.0.1 -U sa -P Secret_password_1
+      interval: 3s
+      timeout: 3s
+      retries: 10
+
+
+  databases:
+    container_name: databases
+    image: docker/whalesay
+    command: >
+      cowsay
+        "Both databases have been started and are ready for TDK exercises!"
+    depends_on:
+      input_db:
+        condition: service_healthy
+      output_db:
+        condition: service_healthy
+
+
+  tdk:
+    extends:
+      file: ../parent-compose.yml
+      service: tdk
+    environment:
+      SYNTHESIZED_INPUT_URL: jdbc:sqlserver://output_db:1433;databaseName=sakila;encrypt=false
+      SYNTHESIZED_INPUT_USERNAME: sa
+      SYNTHESIZED_INPUT_PASSWORD: Secret_password_1
+      SYNTHESIZED_OUTPUT_URL: jdbc:sqlserver://input_db:1433;databaseName=sakila;encrypt=false
+      SYNTHESIZED_OUTPUT_USERNAME: sa
+      SYNTHESIZED_OUTPUT_PASSWORD: Secret_password_1
+    volumes:
+      - ./config.tdk.yaml:/app/config.yaml