From 76f3a760fac405c4c3de9ba673fc8517dbaa6a69 Mon Sep 17 00:00:00 2001 From: mgramin Date: Tue, 20 Feb 2024 16:27:21 +0300 Subject: [PATCH 1/6] Set concrete pagila version --- postgres/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres/docker-compose.yaml b/postgres/docker-compose.yaml index b94cf38..5986473 100644 --- a/postgres/docker-compose.yaml +++ b/postgres/docker-compose.yaml @@ -4,7 +4,7 @@ services: input_db: container_name: input_db - image: synthesizedio/pagila + image: synthesizedio/pagila:1.3 environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} @@ -19,7 +19,7 @@ services: output_db: container_name: output_db - image: postgres:16 + image: synthesizedio/1.3-schema-only environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} From 89c7230214ad354fa1693129f5c82a73535213d1 Mon Sep 17 00:00:00 2001 From: mgramin Date: Tue, 20 Feb 2024 16:29:45 +0300 Subject: [PATCH 2/6] update image name --- postgres/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/docker-compose.yaml b/postgres/docker-compose.yaml index 5986473..83fdc9c 100644 --- a/postgres/docker-compose.yaml +++ b/postgres/docker-compose.yaml @@ -19,7 +19,7 @@ services: output_db: container_name: output_db - image: synthesizedio/1.3-schema-only + image: synthesizedio/pagila:1.3-schema-only environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} From 54089b449eab2811f43d43c8548973c5819bb70a Mon Sep 17 00:00:00 2001 From: mgramin Date: Tue, 20 Feb 2024 16:33:34 +0300 Subject: [PATCH 3/6] Fix tests --- postgres/soda/checks_for_2x_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/soda/checks_for_2x_generation.yaml b/postgres/soda/checks_for_2x_generation.yaml index a08af8a..c066301 100644 --- a/postgres/soda/checks_for_2x_generation.yaml +++ b/postgres/soda/checks_for_2x_generation.yaml @@ -26,7 +26,7 @@ checks for film_actor: - row_count = 10924 checks for film_category: - - row_count = 4638 + - row_count = 4734 checks for inventory: - row_count = 9162 From 2900cc3479e1c9ba9123712f4e769ce19197f54d Mon Sep 17 00:00:00 2001 From: mgramin Date: Tue, 20 Feb 2024 17:01:47 +0300 Subject: [PATCH 4/6] Update pagila to 1.4 --- postgres/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres/docker-compose.yaml b/postgres/docker-compose.yaml index 83fdc9c..7152702 100644 --- a/postgres/docker-compose.yaml +++ b/postgres/docker-compose.yaml @@ -4,7 +4,7 @@ services: input_db: container_name: input_db - image: synthesizedio/pagila:1.3 + image: synthesizedio/pagila:1.4 environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} @@ -19,7 +19,7 @@ services: output_db: container_name: output_db - image: synthesizedio/pagila:1.3-schema-only + image: synthesizedio/pagila:1.4-schema-only environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} From 473304f4b39eaf3351b62e255625351e78eb5ed6 Mon Sep 17 00:00:00 2001 From: mgramin Date: Tue, 20 Feb 2024 20:59:29 +0300 Subject: [PATCH 5/6] Update --- postgres/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres/docker-compose.yaml b/postgres/docker-compose.yaml index 7152702..518d0c3 100644 --- a/postgres/docker-compose.yaml +++ b/postgres/docker-compose.yaml @@ -4,7 +4,7 @@ services: input_db: container_name: input_db - image: synthesizedio/pagila:1.4 + image: synthesizedio/pagila:1.5 environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} @@ -19,7 +19,7 @@ services: output_db: container_name: output_db - image: synthesizedio/pagila:1.4-schema-only + image: postgres:16 environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} From 01f814b0d5c82f5f58752d8eb2026202c6711df7 Mon Sep 17 00:00:00 2001 From: mgramin Date: Tue, 20 Feb 2024 21:04:17 +0300 Subject: [PATCH 6/6] Add new tests --- db2/config_masking.tdk.dot | 41 ++ db2/config_masking.tdk.yaml | 50 +- db2/docker-compose.yaml | 24 +- mssqlserver/config.tdk.yaml | 27 ++ mssqlserver/docker-compose.yaml | 62 +++ oracle/config.tdk.yaml | 170 +++---- oracle/docker-compose.yaml | 44 +- oracle/sakila-schema-output.sql | 795 +++++++++++++++++++++++++++++++ oracle/sakila-schema.sql | 36 +- oracle/test.sql | 10 + parent-compose.yml | 21 + postgres/config_masking.tdk.yaml | 13 +- 12 files changed, 1154 insertions(+), 139 deletions(-) create mode 100644 db2/config_masking.tdk.dot create mode 100644 mssqlserver/config.tdk.yaml create mode 100644 mssqlserver/docker-compose.yaml create mode 100644 oracle/sakila-schema-output.sql create mode 100644 oracle/test.sql diff --git a/db2/config_masking.tdk.dot b/db2/config_masking.tdk.dot new file mode 100644 index 0000000..3ed03e8 --- /dev/null +++ b/db2/config_masking.tdk.dot @@ -0,0 +1,41 @@ +strict digraph G { + 1 [ label="InputDatabase [id=default_source]" shape="folder" ]; + 2 [ label="OutputDatabase [id=default_target]" shape="folder" ]; + 3 [ label="TableMasking [PADT_ORS.C_PADT_ACCOUNT]" shape="ellipse" ]; + 4 [ label="InputTableRead [PADT_ORS.C_PADT_ACCOUNT]" shape="cylinder" ]; + 5 [ label="OutputTableRead [PADT_ORS.C_PADT_ACCOUNT]" shape="cylinder" ]; + 6 [ label="TableStore [PADT_ORS.C_PADT_ACCOUNT]" shape="cylinder" ]; + 7 [ label="TableMasking [PADT_ORS.C_PADT_PARTY]" shape="ellipse" ]; + 8 [ label="InputTableRead [PADT_ORS.C_PADT_PARTY]" shape="cylinder" ]; + 9 [ label="OutputTableRead [PADT_ORS.C_PADT_PARTY]" shape="cylinder" ]; + 10 [ label="TableStore [PADT_ORS.C_PADT_PARTY]" shape="cylinder" ]; + 11 [ label="TableMasking [PADT_ORS.C_PTY_TO_ACCT]" shape="ellipse" ]; + 12 [ label="InputTableRead [PADT_ORS.C_PTY_TO_ACCT]" shape="cylinder" ]; + 13 [ label="TableStore [PADT_ORS.C_PTY_TO_ACCT]" shape="cylinder" ]; + 14 [ label="InputAlignment [PADT_ORS.C_PTY_TO_ACCT]" shape="ellipse" ]; + 15 [ label="TableFilter [streamed=PADT_ORS.C_PTY_TO_ACCT build=PADT_ORS.C_PADT_PARTY,build=PADT_ORS.C_PADT_ACCOUNT]" shape="ellipse" ]; + 16 [ label="TableSample [PADT_ORS.C_PTY_TO_ACCT]" shape="ellipse" ]; + 17 [ label="TableSample [PADT_ORS.C_PADT_ACCOUNT]" shape="ellipse" ]; + 18 [ label="TableSample [PADT_ORS.C_PADT_PARTY]" shape="ellipse" ]; + 1 -> 4 [ label="belongs" style="dotted" arrowhead="crow" ]; + 2 -> 6 [ label="belongs" style="dotted" arrowhead="crow" ]; + 1 -> 8 [ label="belongs" style="dotted" arrowhead="crow" ]; + 2 -> 10 [ label="belongs" style="dotted" arrowhead="crow" ]; + 1 -> 12 [ label="belongs" style="dotted" arrowhead="crow" ]; + 2 -> 13 [ label="belongs" style="dotted" arrowhead="crow" ]; + 12 -> 14 [ label="data_flow" style="solid" arrowhead="normal" ]; + 15 -> 16 [ label="data_flow" style="solid" arrowhead="normal" ]; + 10 -> 9 [ label="await" style="solid" arrowhead="normal" ]; + 9 -> 15 [ label="table_reference" style="solid" arrowhead="normal" ]; + 14 -> 11 [ label="table_reference" style="solid" arrowhead="normal" ]; + 6 -> 5 [ label="await" style="solid" arrowhead="normal" ]; + 5 -> 15 [ label="table_reference" style="solid" arrowhead="normal" ]; + 11 -> 15 [ label="data_flow" style="solid" arrowhead="normal" ]; + 16 -> 13 [ label="data_flow" style="solid" arrowhead="normal" ]; + 3 -> 17 [ label="data_flow" style="solid" arrowhead="normal" ]; + 17 -> 6 [ label="data_flow" style="solid" arrowhead="normal" ]; + 4 -> 3 [ label="data_flow" style="solid" arrowhead="normal" ]; + 7 -> 18 [ label="data_flow" style="solid" arrowhead="normal" ]; + 18 -> 10 [ label="data_flow" style="solid" arrowhead="normal" ]; + 8 -> 7 [ label="data_flow" style="solid" arrowhead="normal" ]; +} diff --git a/db2/config_masking.tdk.yaml b/db2/config_masking.tdk.yaml index 43b7b51..7d6c61b 100644 --- a/db2/config_masking.tdk.yaml +++ b/db2/config_masking.tdk.yaml @@ -1,8 +1,50 @@ default_config: - mode: MASKING + mode: KEEP + use_working_directory: true + # mode: GENERATION + # target_row_number: 5_000_000 + target_ratio: 0.0 + insert_batch_size: 50000 -table_truncation_mode: "TRUNCATE" -schema_creation_mode: DROP_AND_CREATE -cycle_resolution_strategy: "DEFER_FOREIGN_KEY" +tables: + - table_name_with_schema: "PADT_ORS.C_PADT_ACCOUNT" + target_ratio: 0.01 + # filter: "ROWID_OBJECT BETWEEN '0' AND '111I6'" + + - table_name_with_schema: "PADT_ORS.C_PADT_PARTY" + target_ratio: 0.01 + # filter: "ROWID_OBJECT IN (SELECT act.ROWID_PARTY FROM PADT_ORS.C_PTY_TO_ACCT act WHERE act.ROWID_ACCT BETWEEN '0' AND '111I6')" + + - table_name_with_schema: "PADT_ORS.C_PTY_TO_ACCT" + target_ratio: 0.01 + +table_truncation_mode: DO_NOT_TRUNCATE +schema_creation_mode: DO_NOT_CREATE safety_mode: "RELAXED" +schemas: + - "PADT_ORS" + + + +metadata: + tables: + - table_name_with_schema: "PADT_ORS.C_PTY_TO_ACCT" + foreign_keys: + fk_party: + referred_schema: "PADT_ORS" + referred_table: "C_PADT_PARTY" + columns: + - column: "ROWID_PARTY" + referred_column: "ROWID_OBJECT" + fk_account: + referred_schema: "PADT_ORS" + referred_table: "C_PADT_ACCOUNT" + columns: + - column: "ROWID_ACCT" + referred_column: "ROWID_OBJECT" + + + + + diff --git a/db2/docker-compose.yaml b/db2/docker-compose.yaml index d464339..3f8fc8c 100644 --- a/db2/docker-compose.yaml +++ b/db2/docker-compose.yaml @@ -36,15 +36,28 @@ services: start_period: 10s + databases: + container_name: databases + image: docker/whalesay + command: > + cowsay + "Both databases have been started and are ready for TDK exercises!" + depends_on: + input_db: + condition: service_healthy + output_db: + condition: service_healthy + + tdk: extends: file: ../parent-compose.yml service: tdk environment: - SYNTHESIZED_INPUT_URL: jdbc:db2://input_db:50000/SAMPLE + SYNTHESIZED_INPUT_URL: jdbc:db2://output_db:50000/testdb SYNTHESIZED_INPUT_USERNAME: DB2INST1 SYNTHESIZED_INPUT_PASSWORD: test - SYNTHESIZED_OUTPUT_URL: jdbc:db2://output_db:50000/testdb + SYNTHESIZED_OUTPUT_URL: jdbc:db2://input_db:50000/SAMPLE SYNTHESIZED_OUTPUT_USERNAME: DB2INST1 SYNTHESIZED_OUTPUT_PASSWORD: test SYNTHESIZED_USERCONFIG_FILE: /app/config.yaml @@ -52,11 +65,6 @@ services: TDK_WORKINGDIRECTORY_ENABLED: "true" SYNTHESIZED_KEY: ${SYNTHESIZED_KEY} JAVA_TOOL_OPTIONS: > - -Dlogging.level.io.synthesized.tdk.executor.lite.LiteTransformer=INFO - -Dlogging.level.io.synthesized.tdk=WARN - -Dlogging.level.com.zaxxer.hikari=WARN - -Dlogging.level.org.reflections=WARN - -Dlogging.level.org.jooq=WARN -XX:+UseContainerSupport -XX:MaxRAMPercentage=80.0 -Dspring.main.banner-mode=CONSOLE @@ -64,4 +72,4 @@ services: volumes: - ./${CONFIG_FILE:-config_masking.tdk.yaml}:/app/config.yaml - ../banner.txt:/app/banner.txt - - ../logback-lite-executor.xml:/app/logback-lite-executor.xml + # - ../logback-lite-executor.xml:/app/logback-lite-executor.xml diff --git a/mssqlserver/config.tdk.yaml b/mssqlserver/config.tdk.yaml new file mode 100644 index 0000000..f2d5865 --- /dev/null +++ b/mssqlserver/config.tdk.yaml @@ -0,0 +1,27 @@ +default_config: + # mode: MASKING + mode: GENERATION + # target_row_number: 10 + target_ratio: 2 + + +# tables: +# - table_name_with_schema: "dbo.test_date_types_table" +# transformations: +# - columns: +# - "datetime_column" +# params: +# type: "date_time_unique_hashing" +# min: "1926-04-08T03:17:17.786Z" +# max: "2054-03-30T16:54:42.16Z" +# - columns: +# - "datetime2_column" +# params: +# type: "date_time_unique_hashing" +# min: "1926-04-08T03:17:17.786Z" +# max: "2054-03-30T16:54:42.16Z" + + +table_truncation_mode: "TRUNCATE" +schema_creation_mode: DO_NOT_CREATE +safety_mode: "RELAXED" diff --git a/mssqlserver/docker-compose.yaml b/mssqlserver/docker-compose.yaml new file mode 100644 index 0000000..d75bd4f --- /dev/null +++ b/mssqlserver/docker-compose.yaml @@ -0,0 +1,62 @@ +version: '3' + +services: + + input_db: + container_name: input_db + image: mcr.microsoft.com/mssql/server:2017-latest + ports: + - 6000:1433 + environment: + MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Secret_password_1} + ACCEPT_EULA: Y + healthcheck: + test: > + echo 'select 1;' | /opt/mssql-tools/bin/sqlcmd -S 127.0.0.1 -U sa -P Secret_password_1 + interval: 3s + timeout: 3s + retries: 10 + + + output_db: + container_name: output_db + image: mcr.microsoft.com/mssql/server:2017-latest + ports: + - 6001:1433 + environment: + MSSQL_SA_PASSWORD: ${MSSQL_SA_PASSWORD:-Secret_password_1} + ACCEPT_EULA: Y + healthcheck: + test: > + echo 'select 1;' | /opt/mssql-tools/bin/sqlcmd -S 127.0.0.1 -U sa -P Secret_password_1 + interval: 3s + timeout: 3s + retries: 10 + + + databases: + container_name: databases + image: docker/whalesay + command: > + cowsay + "Both databases have been started and are ready for TDK exercises!" + depends_on: + input_db: + condition: service_healthy + output_db: + condition: service_healthy + + + tdk: + extends: + file: ../parent-compose.yml + service: tdk + environment: + SYNTHESIZED_INPUT_URL: jdbc:sqlserver://output_db:1433;databaseName=sakila;encrypt=false + SYNTHESIZED_INPUT_USERNAME: sa + SYNTHESIZED_INPUT_PASSWORD: Secret_password_1 + SYNTHESIZED_OUTPUT_URL: jdbc:sqlserver://input_db:1433;databaseName=sakila;encrypt=false + SYNTHESIZED_OUTPUT_USERNAME: sa + SYNTHESIZED_OUTPUT_PASSWORD: Secret_password_1 + volumes: + - ./config.tdk.yaml:/app/config.yaml diff --git a/oracle/config.tdk.yaml b/oracle/config.tdk.yaml index 66f464f..26b76f7 100644 --- a/oracle/config.tdk.yaml +++ b/oracle/config.tdk.yaml @@ -1,18 +1,21 @@ default_config: - mode: MASKING + mode: KEEP + # mode: GENERATION + # target_ratio: 0 tables: - - table_name_with_schema: "TEST.CITY" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.CITY" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - table_name_with_schema: "TEST.CUSTOMER" + # target_row_number: 5000 transformations: - columns: [ "CREATE_DATE" ] params: @@ -29,85 +32,88 @@ tables: min: 2022-01-01T12:00:00Z max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.FILM" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.STORE" + # target_row_number: 5000 - - table_name_with_schema: "TEST.FILM_ACTOR" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.FILM" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.ACTOR" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.FILM_ACTOR" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.FILM_CATEGORY" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.ACTOR" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.INVENTORY" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.FILM_CATEGORY" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.PAYMENT" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.INVENTORY" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.RENTAL" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.PAYMENT" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z - - table_name_with_schema: "TEST.ADDRESS" - transformations: - - columns: [ "LAST_UPDATE" ] - params: - type: "date_generator" - mean: 2022-03-01T12:00:00Z - std: 7776000000 - min: 2022-01-01T12:00:00Z - max: 2022-07-01T12:00:00Z + # - table_name_with_schema: "TEST.RENTAL" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z + + # - table_name_with_schema: "TEST.ADDRESS" + # transformations: + # - columns: [ "LAST_UPDATE" ] + # params: + # type: "date_generator" + # mean: 2022-03-01T12:00:00Z + # std: 7776000000 + # min: 2022-01-01T12:00:00Z + # max: 2022-07-01T12:00:00Z cycle_resolution_strategy: DEFER_FOREIGN_KEY diff --git a/oracle/docker-compose.yaml b/oracle/docker-compose.yaml index e9788d6..52b78ad 100644 --- a/oracle/docker-compose.yaml +++ b/oracle/docker-compose.yaml @@ -2,44 +2,47 @@ version: '3' services: - oracle_source: + input_db: image: gvenzl/oracle-xe - container_name: oracle_source + container_name: input_db networks: - simple-network ports: - - 1521:1521 + - 6000:1521 environment: ORACLE_PASSWORD: ${ORACLE_USER:-secret_password} volumes: - ./create_user.sql:/container-entrypoint-initdb.d/1.sql - ./sakila-schema.sql:/container-entrypoint-initdb.d/2.sql - ./oracle-sakila-insert-data.sql:/container-entrypoint-initdb.d/3.sql + - ./test.sql:/opt/test.sql healthcheck: - test: "${ORACLE_BASE}/healthcheck.sh" + test: "echo exit | sqlplus / as sysdba @/opt/test.sql" interval: "3s" timeout: "3s" start_period: "3s" retries: 30 - oracle_target: + + output_db: image: gvenzl/oracle-xe - container_name: oracle_target + container_name: output_db networks: - simple-network ports: - - 1522:1521 + - 6001:1521 environment: ORACLE_PASSWORD: ${ORACLE_USER:-secret_password} volumes: - ./create_user.sql:/container-entrypoint-initdb.d/1.sql healthcheck: - test: "${ORACLE_BASE}/healthcheck.sh" + test: "echo 'select 1 from dual;' | sqlplus / as sysdba" interval: "3s" timeout: "3s" start_period: "3s" retries: 30 + databases: container_name: databases image: docker/whalesay @@ -47,9 +50,9 @@ services: cowsay "Both databases have been started and are ready for TDK exercises!" depends_on: - oracle_source: + input_db: condition: service_healthy - oracle_target: + output_db: condition: service_healthy @@ -60,28 +63,19 @@ services: networks: - simple-network environment: - SYNTHESIZED_INPUT_URL: jdbc:oracle:thin:@oracle_source:1521:xe + SYNTHESIZED_INPUT_URL: jdbc:oracle:thin:@output_db:1521:xe SYNTHESIZED_INPUT_USERNAME: test SYNTHESIZED_INPUT_PASSWORD: test - SYNTHESIZED_OUTPUT_URL: jdbc:oracle:thin:@oracle_target:1521:xe + SYNTHESIZED_OUTPUT_URL: jdbc:oracle:thin:@input_db:1521:xe SYNTHESIZED_OUTPUT_USERNAME: test SYNTHESIZED_OUTPUT_PASSWORD: test - SYNTHESIZED_USERCONFIG_FILE: /app/config.yaml - TDK_WORKINGDIRECTORY_PATH: /app/data - SYNTHESIZED_KEY: ${SYNTHESIZED_KEY} # TDK_WORKINGDIRECTORY_ENABLED: "true" - JAVA_TOOL_OPTIONS: > - -Dlogging.level.io.synthesized.tdk.executor.lite.LiteTransformer=INFO - -Dlogging.level.io.synthesized.tdk=WARN - -Dlogging.level.com.zaxxer.hikari=WARN - -Dlogging.level.org.reflections=WARN - -XX:+UseContainerSupport - -XX:MaxRAMPercentage=80.0 - -Dspring.main.banner-mode=CONSOLE - -Dspring.banner.location=file:/app/banner.txt + SYNTHESIZED_COMMAND: --graph-dump-path plan.dot volumes: - ./config.tdk.yaml:/app/config.yaml - - ./banner.txt:/app/banner.txt + depends_on: + databases: + condition: service_completed_successfully ora2pg: diff --git a/oracle/sakila-schema-output.sql b/oracle/sakila-schema-output.sql new file mode 100644 index 0000000..ebd55fc --- /dev/null +++ b/oracle/sakila-schema-output.sql @@ -0,0 +1,795 @@ +conn test/test + +/* + +Sakila for Oracle is a port of the Sakila example database available for MySQL, which was originally developed by Mike Hillyer of the MySQL AB documentation team. +This project is designed to help database administrators to decide which database to use for development of new products +The user can run the same SQL against different kind of databases and compare the performance + +License: BSD +Copyright DB Software Laboratory +http://www.etl-tools.com + +*/ + + +-- +-- Table structure for table actor +-- +--DROP TABLE actor; + +CREATE TABLE actor ( + actor_id INT NOT NULL , + first_name VARCHAR(45) NOT NULL, + last_name VARCHAR(45) NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_actor PRIMARY KEY (actor_id) +); + +CREATE INDEX idx_actor_last_name ON actor(last_name); +/ + + --DROP SEQUENCE actor_sequence; + +CREATE SEQUENCE actor_sequence; +/ + + + +CREATE OR REPLACE TRIGGER actor_before_trigger +BEFORE INSERT ON actor FOR EACH ROW +BEGIN + IF (:NEW.actor_id IS NULL) THEN + SELECT actor_sequence.nextval INTO :NEW.actor_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER actor_before_update +BEFORE UPDATE ON actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + + +-- +-- Table structure for table country +-- + +CREATE TABLE country ( + country_id INT NOT NULL, + country VARCHAR(50) NOT NULL, + last_update DATE, + CONSTRAINT pk_country PRIMARY KEY (country_id) +); + +---DROP SEQUENCE country_sequence; + +CREATE SEQUENCE country_sequence; +/ + + +CREATE OR REPLACE TRIGGER country_before_trigger +BEFORE INSERT ON country FOR EACH ROW +BEGIN + IF (:NEW.country_id IS NULL) THEN + SELECT country_sequence.nextval INTO :NEW.country_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER country_before_update +BEFORE UPDATE ON country FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + + +-- +-- Table structure for table city +-- + +CREATE TABLE city ( + city_id INT NOT NULL, + city VARCHAR(50) NOT NULL, + country_id INT NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_city PRIMARY KEY (city_id), + CONSTRAINT fk_city_country FOREIGN KEY (country_id) REFERENCES country (country_id) +); + +CREATE INDEX idx_fk_country_id ON city(country_id); +/ + +--- DROP SEQUENCE city_sequence; + +CREATE SEQUENCE city_sequence; +/ + +CREATE OR REPLACE TRIGGER city_before_trigger +BEFORE INSERT ON city FOR EACH ROW +BEGIN + IF (:NEW.city_id IS NULL) THEN + SELECT city_sequence.nextval INTO :NEW.city_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER city_before_update +BEFORE UPDATE ON city FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + + +-- +-- Table structure for table address +-- + +CREATE TABLE address ( + address_id INT NOT NULL, + address VARCHAR(50) NOT NULL, + address2 VARCHAR(50) DEFAULT NULL, + district VARCHAR(20) NOT NULL, + city_id INT NOT NULL, + postal_code VARCHAR(10) DEFAULT NULL, + phone VARCHAR(20) NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_address PRIMARY KEY (address_id) +); + +CREATE INDEX idx_fk_city_id ON address(city_id); +/ + +ALTER TABLE address ADD CONSTRAINT fk_address_city FOREIGN KEY (city_id) REFERENCES city (city_id); +/ + + --DROP SEQUENCE city_sequence; + +CREATE SEQUENCE address_sequence; +/ + +CREATE OR REPLACE TRIGGER address_before_trigger +BEFORE INSERT ON address FOR EACH ROW +BEGIN + IF (:NEW.address_id IS NULL) THEN + SELECT address_sequence.nextval INTO :NEW.address_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER address_before_update +BEFORE UPDATE ON address FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table language +-- + +CREATE TABLE language ( + language_id INT NOT NULL , + name CHAR(20) NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_language PRIMARY KEY (language_id) +); + +---DROP SEQUENCE language_sequence; + +CREATE SEQUENCE language_sequence; +/ + +CREATE OR REPLACE TRIGGER language_before_trigger +BEFORE INSERT ON language FOR EACH ROW +BEGIN + IF (:NEW.language_id IS NULL) THEN + SELECT language_sequence.nextval INTO :NEW.language_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER language_before_update +BEFORE UPDATE ON language FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table category +-- + +CREATE TABLE category ( + category_id INT NOT NULL, + name VARCHAR(25) NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_category PRIMARY KEY (category_id) +); + +---DROP SEQUENCE category_sequence; + +CREATE SEQUENCE category_sequence; +/ + +CREATE OR REPLACE TRIGGER category_before_trigger +BEFORE INSERT ON category FOR EACH ROW +BEGIN + IF (:NEW.category_id IS NULL) THEN + SELECT category_sequence.nextval INTO :NEW.category_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER category_before_update +BEFORE UPDATE ON category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table customer +-- + +CREATE TABLE customer ( + customer_id INT NOT NULL, + store_id INT NOT NULL, + first_name VARCHAR(45) NOT NULL, + last_name VARCHAR(45) NOT NULL, + email VARCHAR(50) DEFAULT NULL, + address_id INT NOT NULL, + active CHAR(1) DEFAULT 'Y' NOT NULL, + create_date DATE NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_customer PRIMARY KEY (customer_id), + CONSTRAINT fk_customer_address FOREIGN KEY (address_id) REFERENCES address(address_id) +); + +CREATE INDEX idx_customer_fk_store_id ON customer(store_id); +/ +CREATE INDEX idx_customer_fk_address_id ON customer(address_id); +/ +CREATE INDEX idx_customer_last_name ON customer(last_name); +/ +---DROP SEQUENCE customer_sequence; + +CREATE SEQUENCE customer_sequence; +/ + +CREATE OR REPLACE TRIGGER customer_before_trigger +BEFORE INSERT ON customer FOR EACH ROW +BEGIN + IF (:NEW.customer_id IS NULL) THEN + SELECT customer_sequence.nextval INTO :NEW.customer_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; + :NEW.create_date:=current_date; +END; +/ +CREATE OR REPLACE TRIGGER customer_before_update +BEFORE UPDATE ON customer FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ +-- +-- Table structure for table film +-- + +CREATE TABLE film ( + film_id INT NOT NULL, + title VARCHAR(255) NOT NULL, + description CLOB DEFAULT NULL, + release_year VARCHAR(4) DEFAULT NULL, + language_id INT NOT NULL, + original_language_id INT DEFAULT NULL, + rental_duration SMALLINT DEFAULT 3 NOT NULL, + rental_rate DECIMAL(4,2) DEFAULT 4.99 NOT NULL, + length SMALLINT DEFAULT NULL, + replacement_cost DECIMAL(5,2) DEFAULT 19.99 NOT NULL, + rating VARCHAR(10) DEFAULT 'G', + special_features VARCHAR(100) DEFAULT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_film PRIMARY KEY (film_id), + CONSTRAINT fk_film_language FOREIGN KEY (language_id) REFERENCES language (language_id) , + CONSTRAINT fk_film_language_original FOREIGN KEY (original_language_id) REFERENCES language (language_id) +); + +ALTER TABLE film ADD CONSTRAINT CHECK_special_features CHECK(special_features is null or + special_features like '%Trailers%' or + special_features like '%Commentaries%' or + special_features like '%Deleted Scenes%' or + special_features like '%Behind the Scenes%'); +/ +ALTER TABLE film ADD CONSTRAINT CHECK_special_rating CHECK(rating in ('G','PG','PG-13','R','NC-17')); +/ +CREATE INDEX idx_fk_language_id ON film(language_id); +/ +CREATE INDEX idx_fk_original_language_id ON film(original_language_id); +/ + +---DROP SEQUENCE film_sequence; + +CREATE SEQUENCE film_sequence; +/ + +CREATE OR REPLACE TRIGGER film_before_trigger +BEFORE INSERT ON film FOR EACH ROW +BEGIN + IF (:NEW.film_id IS NULL) THEN + SELECT film_sequence.nextval INTO :NEW.film_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_before_update +BEFORE UPDATE ON film FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table film_actor +-- + +CREATE TABLE film_actor ( + actor_id INT NOT NULL, + film_id INT NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_film_actor PRIMARY KEY (actor_id,film_id), + CONSTRAINT fk_film_actor_actor FOREIGN KEY (actor_id) REFERENCES actor (actor_id), + CONSTRAINT fk_film_actor_film FOREIGN KEY (film_id) REFERENCES film (film_id) +); + +CREATE INDEX idx_fk_film_actor_film ON film_actor(film_id); +/ + +CREATE INDEX idx_fk_film_actor_actor ON film_actor(actor_id) ; +/ + +CREATE OR REPLACE TRIGGER film_actor_before_trigger +BEFORE INSERT ON film_actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_actor_before_update +BEFORE UPDATE ON film_actor FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table film_category +-- + +CREATE TABLE film_category ( + film_id INT NOT NULL, + category_id INT NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_film_category PRIMARY KEY (film_id, category_id), + CONSTRAINT fk_film_category_film FOREIGN KEY (film_id) REFERENCES film (film_id), + CONSTRAINT fk_film_category_category FOREIGN KEY (category_id) REFERENCES category (category_id) +); + +CREATE INDEX idx_fk_film_category_film ON film_category(film_id); +/ +CREATE INDEX idx_fk_film_category_category ON film_category(category_id); +/ + +CREATE OR REPLACE TRIGGER film_category_before_trigger +BEFORE INSERT ON film_category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER film_category_before_update +BEFORE UPDATE ON film_category FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ +-- +-- Table structure for table film_text +-- + +CREATE TABLE film_text ( + film_id INT NOT NULL, + title VARCHAR(255) NOT NULL, + description CLOB, + CONSTRAINT pk_film_text PRIMARY KEY (film_id) +); + +-- +-- Table structure for table inventory +-- + +CREATE TABLE inventory ( + inventory_id INT NOT NULL, + film_id INT NOT NULL, + store_id INT NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_inventory PRIMARY KEY (inventory_id), + CONSTRAINT fk_inventory_film FOREIGN KEY (film_id) REFERENCES film (film_id) +); + +CREATE INDEX idx_fk_film_id ON inventory(film_id); +/ + +CREATE INDEX idx_fk_film_id_store_id ON inventory(store_id,film_id); +/ + +---DROP SEQUENCE inventory_sequence; + +CREATE SEQUENCE inventory_sequence; +/ + +CREATE OR REPLACE TRIGGER inventory_before_trigger +BEFORE INSERT ON inventory FOR EACH ROW +BEGIN + IF (:NEW.inventory_id IS NULL) THEN + SELECT inventory_sequence.nextval INTO :NEW.inventory_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ +CREATE OR REPLACE TRIGGER inventory_before_update +BEFORE UPDATE ON inventory FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table staff +-- + +CREATE TABLE staff ( + staff_id INT NOT NULL, + first_name VARCHAR(45) NOT NULL, + last_name VARCHAR(45) NOT NULL, + address_id INT NOT NULL, + picture BLOB DEFAULT NULL, + email VARCHAR(50) DEFAULT NULL, + store_id INT NOT NULL, + active SMALLINT DEFAULT 1 NOT NULL, + username VARCHAR(16) NOT NULL, + password VARCHAR(40) DEFAULT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_staff PRIMARY KEY (staff_id), + CONSTRAINT fk_staff_address FOREIGN KEY (address_id) REFERENCES address (address_id) +); + +CREATE INDEX idx_fk_staff_store_id ON staff(store_id); +/ + +CREATE INDEX idx_fk_staff_address_id ON staff(address_id); +/ + +---DROP SEQUENCE inventory_sequence; + +CREATE SEQUENCE staff_sequence; +/ + +CREATE OR REPLACE TRIGGER staff_before_trigger +BEFORE INSERT ON staff FOR EACH ROW +BEGIN + IF (:NEW.staff_id IS NULL) THEN + SELECT staff_sequence.nextval INTO :NEW.staff_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER staff_before_update +BEFORE UPDATE ON staff FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table store +-- + +CREATE TABLE store ( + store_id INT NOT NULL, + manager_staff_id INT NOT NULL, + address_id INT NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_store PRIMARY KEY (store_id), + CONSTRAINT fk_store_staff FOREIGN KEY (manager_staff_id) REFERENCES staff (staff_id) , + CONSTRAINT fk_store_address FOREIGN KEY (address_id) REFERENCES address (address_id) +); + +CREATE INDEX idx_store_fk_manager_staff_id ON store(manager_staff_id); +/ + +CREATE INDEX idx_fk_store_address ON store(address_id); +/ + +---DROP SEQUENCE store_sequence; + +CREATE SEQUENCE store_sequence; +/ + + + +CREATE OR REPLACE TRIGGER store_before_trigger +BEFORE INSERT ON store FOR EACH ROW +BEGIN + IF (:NEW.store_id IS NULL) THEN + SELECT store_sequence.nextval INTO :NEW.store_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER store_before_update +BEFORE UPDATE ON store FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- +-- Table structure for table payment +-- + +CREATE TABLE payment ( + payment_id INT NOT NULL, + customer_id INT NOT NULL, + staff_id INT NOT NULL, + rental_id INT DEFAULT NULL, + amount DECIMAL(5,2) NOT NULL, + payment_date DATE NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_payment PRIMARY KEY (payment_id), + CONSTRAINT fk_payment_customer FOREIGN KEY (customer_id) REFERENCES customer (customer_id) , + CONSTRAINT fk_payment_staff FOREIGN KEY (staff_id) REFERENCES staff (staff_id) +); + +CREATE INDEX idx_fk_staff_id ON payment(staff_id); +/ +CREATE INDEX idx_fk_customer_id ON payment(customer_id); +/ + +---DROP SEQUENCE payment_sequence; + +CREATE SEQUENCE payment_sequence; +/ + +CREATE OR REPLACE TRIGGER payment_before_trigger +BEFORE INSERT ON payment FOR EACH ROW +BEGIN + IF (:NEW.payment_id IS NULL) THEN + SELECT payment_sequence.nextval INTO :NEW.payment_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER payment_before_update +BEFORE UPDATE ON payment FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +CREATE TABLE rental ( + rental_id INT NOT NULL, + rental_date DATE NOT NULL, + inventory_id INT NOT NULL, + customer_id INT NOT NULL, + return_date DATE DEFAULT NULL, + staff_id INT NOT NULL, + last_update DATE NOT NULL, + CONSTRAINT pk_rental PRIMARY KEY (rental_id), + CONSTRAINT fk_rental_staff FOREIGN KEY (staff_id) REFERENCES staff (staff_id) , + CONSTRAINT fk_rental_inventory FOREIGN KEY (inventory_id) REFERENCES inventory (inventory_id) , + CONSTRAINT fk_rental_customer FOREIGN KEY (customer_id) REFERENCES customer (customer_id) +); + +CREATE INDEX idx_rental_fk_inventory_id ON rental(inventory_id); +/ +CREATE INDEX idx_rental_fk_customer_id ON rental(customer_id); +/ +CREATE INDEX idx_rental_fk_staff_id ON rental(staff_id); +/ +CREATE UNIQUE INDEX idx_rental_uq ON rental (rental_date,inventory_id,customer_id); +/ + +---DROP SEQUENCE payment_sequence; + +CREATE SEQUENCE rental_sequence; +/ + +CREATE OR REPLACE TRIGGER rental_before_trigger +BEFORE INSERT ON rental FOR EACH ROW +BEGIN + IF (:NEW.rental_id IS NULL) THEN + SELECT rental_sequence.nextval INTO :NEW.rental_id + FROM DUAL; + END IF; + :NEW.last_update:=current_date; +END; +/ + +CREATE OR REPLACE TRIGGER rental_before_update +BEFORE UPDATE ON rental FOR EACH ROW +BEGIN + :NEW.last_update:=current_date; +END; +/ + +-- FK CONSTRAINTS +ALTER TABLE customer ADD CONSTRAINT fk_customer_store FOREIGN KEY (store_id) REFERENCES store (store_id); +/ +ALTER TABLE inventory ADD CONSTRAINT fk_inventory_store FOREIGN KEY (store_id) REFERENCES store (store_id); +/ +ALTER TABLE staff ADD CONSTRAINT fk_staff_store FOREIGN KEY (store_id) REFERENCES store (store_id); +/ +ALTER TABLE payment ADD CONSTRAINT fk_payment_rental FOREIGN KEY (rental_id) REFERENCES rental (rental_id) ON DELETE SET NULL; +/ +-- +-- View structure for view customer_list +-- + +CREATE OR REPLACE VIEW customer_list +AS +SELECT cu.customer_id AS ID, + cu.first_name||' '||cu.last_name AS name, + a.address AS address, + a.postal_code AS zip_code, + a.phone AS phone, + city.city AS city, + country.country AS country, + decode(cu.active, 1,'active','') AS notes, + cu.store_id AS SID +FROM customer cu JOIN address a ON cu.address_id = a.address_id JOIN city ON a.city_id = city.city_id + JOIN country ON city.country_id = country.country_id; +/ +-- +-- View structure for view film_list +-- + +CREATE OR REPLACE VIEW film_list +AS +SELECT film.film_id AS FID, + film.title AS title, + film.description AS description, + category.name AS category, + film.rental_rate AS price, + film.length AS length, + film.rating AS rating, + actor.first_name||' '||actor.last_name AS actors +FROM category LEFT JOIN film_category ON category.category_id = film_category.category_id LEFT JOIN film ON film_category.film_id = film.film_id + JOIN film_actor ON film.film_id = film_actor.film_id + JOIN actor ON film_actor.actor_id = actor.actor_id; +/ + +-- +-- View structure for view staff_list +-- + +CREATE OR REPLACE VIEW staff_list +AS +SELECT s.staff_id AS ID, + s.first_name||' '||s.last_name AS name, + a.address AS address, + a.postal_code AS zip_code, + a.phone AS phone, + city.city AS city, + country.country AS country, + s.store_id AS SID +FROM staff s JOIN address a ON s.address_id = a.address_id JOIN city ON a.city_id = city.city_id + JOIN country ON city.country_id = country.country_id; +/ +-- +-- View structure for view sales_by_store +-- + +CREATE OR REPLACE VIEW sales_by_store +AS +SELECT + s.store_id + ,c.city||','||cy.country AS store + ,m.first_name||' '||m.last_name AS manager + ,SUM(p.amount) AS total_sales +FROM payment p +INNER JOIN rental r ON p.rental_id = r.rental_id +INNER JOIN inventory i ON r.inventory_id = i.inventory_id +INNER JOIN store s ON i.store_id = s.store_id +INNER JOIN address a ON s.address_id = a.address_id +INNER JOIN city c ON a.city_id = c.city_id +INNER JOIN country cy ON c.country_id = cy.country_id +INNER JOIN staff m ON s.manager_staff_id = m.staff_id +GROUP BY + s.store_id +, c.city||','||cy.country +, m.first_name||' '||m.last_name; +/ +-- +-- View structure for view sales_by_film_category +-- +-- Note that total sales will add up to >100% because +-- some titles belong to more than 1 category +-- + +CREATE OR REPLACE VIEW sales_by_film_category +AS +SELECT +c.name AS category +, SUM(p.amount) AS total_sales +FROM payment p +INNER JOIN rental r ON p.rental_id = r.rental_id +INNER JOIN inventory i ON r.inventory_id = i.inventory_id +INNER JOIN film f ON i.film_id = f.film_id +INNER JOIN film_category fc ON f.film_id = fc.film_id +INNER JOIN category c ON fc.category_id = c.category_id +GROUP BY c.name; +/ + +-- +-- View structure for view actor_info +-- + +/* +CREATE VIEW actor_info +AS +SELECT +a.actor_id, +a.first_name, +a.last_name, +GROUP_CONCAT(DISTINCT CONCAT(c.name, ': ', + (SELECT GROUP_CONCAT(f.title ORDER BY f.title SEPARATOR ', ') + FROM sakila.film f + INNER JOIN sakila.film_category fc + ON f.film_id = fc.film_id + INNER JOIN sakila.film_actor fa + ON f.film_id = fa.film_id + WHERE fc.category_id = c.category_id + AND fa.actor_id = a.actor_id + ) + ) + ORDER BY c.name SEPARATOR '; ') +AS film_info +FROM sakila.actor a +LEFT JOIN sakila.film_actor fa + ON a.actor_id = fa.actor_id +LEFT JOIN sakila.film_category fc + ON fa.film_id = fc.film_id +LEFT JOIN sakila.category c + ON fc.category_id = c.category_id +GROUP BY a.actor_id, a.first_name, a.last_name; +*/ + +-- TO DO PROCEDURES diff --git a/oracle/sakila-schema.sql b/oracle/sakila-schema.sql index ebd55fc..9b3f182 100644 --- a/oracle/sakila-schema.sql +++ b/oracle/sakila-schema.sql @@ -36,23 +36,23 @@ CREATE SEQUENCE actor_sequence; -CREATE OR REPLACE TRIGGER actor_before_trigger -BEFORE INSERT ON actor FOR EACH ROW -BEGIN - IF (:NEW.actor_id IS NULL) THEN - SELECT actor_sequence.nextval INTO :NEW.actor_id - FROM DUAL; - END IF; - :NEW.last_update:=current_date; -END; -/ - -CREATE OR REPLACE TRIGGER actor_before_update -BEFORE UPDATE ON actor FOR EACH ROW -BEGIN - :NEW.last_update:=current_date; -END; -/ +-- CREATE OR REPLACE TRIGGER actor_before_trigger +-- BEFORE INSERT ON actor FOR EACH ROW +-- BEGIN +-- IF (:NEW.actor_id IS NULL) THEN +-- SELECT actor_sequence.nextval INTO :NEW.actor_id +-- FROM DUAL; +-- END IF; +-- :NEW.last_update:=current_date; +-- END; +-- / + +-- CREATE OR REPLACE TRIGGER actor_before_update +-- BEFORE UPDATE ON actor FOR EACH ROW +-- BEGIN +-- :NEW.last_update:=current_date; +-- END; +-- / -- @@ -519,7 +519,7 @@ END; -- CREATE TABLE store ( - store_id INT NOT NULL, + store_id INT, --NOT NULL, manager_staff_id INT NOT NULL, address_id INT NOT NULL, last_update DATE NOT NULL, diff --git a/oracle/test.sql b/oracle/test.sql new file mode 100644 index 0000000..a6bcf46 --- /dev/null +++ b/oracle/test.sql @@ -0,0 +1,10 @@ +DECLARE + ex_custom EXCEPTION; + payments_count NUMBER; +BEGIN + SELECT COUNT(*) INTO payments_count FROM test.payment; + IF payments_count != 16049 THEN + raise_application_error( -20001, 'Tables are empty' ); + END IF; +END; +/ diff --git a/parent-compose.yml b/parent-compose.yml index 1cce804..929c8cc 100644 --- a/parent-compose.yml +++ b/parent-compose.yml @@ -4,3 +4,24 @@ services: tdk: container_name: tdk image: synthesizedio/synthesized-tdk-cli:v1.44.0 + environment: + SYNTHESIZED_KEY: ${SYNTHESIZED_KEY} + SYNTHESIZED_USERCONFIG_FILE: /app/config.yaml + TDK_WORKINGDIRECTORY_PATH: /app/data + # SYNTHESIZED_COMMAND: dry-run + JAVA_TOOL_OPTIONS: > + -Dlogging.level.io.synthesized.tdk.executor.lite.AssistedTableLookupFilter=DEBUG + # -Dlogging.level.io.synthesized.tdk.executor.lite.LiteTransformer=INFO + # -Dlogging.level.io.synthesized.tdk=WARN + # -Dlogging.level.com.zaxxer.hikari=WARN + # -Dlogging.level.org.reflections=WARN + # -XX:+UseContainerSupport + # -XX:MaxRAMPercentage=80.0 + # -Dspring.main.banner-mode=CONSOLE + # -Dspring.banner.location=file:/app/banner.txt + volumes: + - ./banner.txt:/app/banner.txt + # - ./logback-lite-executor.xml:/app/logback-lite-executor.xml + # depends_on: + # databases: + # condition: service_completed_successfully diff --git a/postgres/config_masking.tdk.yaml b/postgres/config_masking.tdk.yaml index f4573d8..b167159 100644 --- a/postgres/config_masking.tdk.yaml +++ b/postgres/config_masking.tdk.yaml @@ -1,5 +1,14 @@ default_config: - mode: MASKING + mode: KEEP -schema_creation_mode: DROP_AND_CREATE +# tables: +# - table_name_with_schema: "public.staff" +# transformations: +# - columns: +# - "first_name" +# params: + + +schema_creation_mode: CREATE_IF_NOT_EXISTS safety_mode: "RELAXED" +table_truncation_mode: TRUNCATE