From 6e90ca7d45561056917798f707a282a8fe0cbdc8 Mon Sep 17 00:00:00 2001 From: Vitalii Shilin Date: Thu, 27 Jun 2024 15:37:54 +0200 Subject: [PATCH 1/5] Create creating_test_table_specified_mem_object_size.md Article based on the https://exasol.atlassian.net/browse/IPS-503 --- ...ng_test_table_specified_mem_object_size.md | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 Database-Features/creating_test_table_specified_mem_object_size.md diff --git a/Database-Features/creating_test_table_specified_mem_object_size.md b/Database-Features/creating_test_table_specified_mem_object_size.md new file mode 100644 index 0000000..c240c89 --- /dev/null +++ b/Database-Features/creating_test_table_specified_mem_object_size.md @@ -0,0 +1,87 @@ +## Creating a test table with a specified MEM_OBJECT_SIZE + +This article provides a detailed guide on using the create_table_mem_object_size script to create tables with a specified memory size (MEM_OBJECT_SIZE). It might be useful for Support investigations and experiments. + +To use the script, simply create it in your desired schema. Being careful if a table named "dummy" already exists as the script will recreate it. +The script is designed to create a target table where each row occupies 128 bytes, with randomly generated content ensuring the actual table mem size is 128 bytes times the number of rows. +The number of rows required to populate the target table is calculated based on the value of the parameter mem_object_size. +To generate a potentially large target table, the script uses a cartesian join of a smaller manageable "dummy" table on itself, multiplying the record count to reach the desired size. + +# script +```sql +create schema test_data_gen; +open schema test_data_gen; + +--/ +create or replace script create_table_mem_object_size( + schema_name, table_name, --target table schema/name + mem_object_size --target table desired mem_object_size + ) as +dummy_table_rows_cnt = (math.floor(math.sqrt(mem_object_size * 1024 * 1024 / 128))) + 1 + +res1 = query([[ +DROP TABLE IF EXISTS dummy; +]]) + +res2 = query([[ +CREATE TABLE dummy as +SELECT 1 AS dummy FROM dual CONNECT BY LEVEL <= ]] .. dummy_table_rows_cnt .. [[; +]]) + +res3 = query([[ +CREATE TABLE ]] .. schema_name .. '.' .. table_name .. [[ as +SELECT + RAND(1,100) AS rand1, + RAND(1,100) AS rand2, + RAND(1,100) AS rand3, + RAND(1,100) AS rand4, + RAND(1,100) AS rand5, + RAND(1,100) AS rand6, + RAND(1,100) AS rand7, + RAND(1,100) AS rand8, + RAND(1,100) AS rand9, + RAND(1,100) AS rand10, + RAND(1,100) AS rand11, + RAND(1,100) AS rand12, + RAND(1,100) AS rand13, + RAND(1,100) AS rand14, + RAND(1,100) AS rand15, + RAND(1,100) AS rand16 + + --each row takes 128 byte +FROM +(SELECT 1 from +test_data_gen.dummy x1, +test_data_gen.dummy x2); +]]) +/ +``` + +# Examples + +1. 100Mb +```sql +EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_100mb', 100) WITH OUTPUT; +--2 second on a basic singlenode vm cluster + +select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_100MB' and ROOT_NAME = 'TEST_DATA_GEN' +--result 100.22835159301757812500 +``` + +2. 2Gb +```sql +EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_2gb', 2048) WITH OUTPUT; +--40 second on a basic singlenode vm cluster + +select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_2GB' and ROOT_NAME = 'TEST_DATA_GEN' +--result 2049.37433147430419921875 +``` + +3. 10Gb +```sql +EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_10gb', 10240) WITH OUTPUT; +--4 minutes on a basic singlenode vm cluster + +select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_10GB' and ROOT_NAME = 'TEST_DATA_GEN' +--result 10240.29162216186523437500 +``` From 59402ddf734d7b0267d5bd8ca4a788c391070d19 Mon Sep 17 00:00:00 2001 From: Vitalii Shilin Date: Thu, 27 Jun 2024 16:02:34 +0200 Subject: [PATCH 2/5] Update creating_test_table_specified_mem_object_size.md --- .../creating_test_table_specified_mem_object_size.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Database-Features/creating_test_table_specified_mem_object_size.md b/Database-Features/creating_test_table_specified_mem_object_size.md index c240c89..d94a739 100644 --- a/Database-Features/creating_test_table_specified_mem_object_size.md +++ b/Database-Features/creating_test_table_specified_mem_object_size.md @@ -2,7 +2,7 @@ This article provides a detailed guide on using the create_table_mem_object_size script to create tables with a specified memory size (MEM_OBJECT_SIZE). It might be useful for Support investigations and experiments. -To use the script, simply create it in your desired schema. Being careful if a table named "dummy" already exists as the script will recreate it. +To use the script, simply create it in your desired schema. Be careful if a table named "dummy" already exists as the script will recreate it. The script is designed to create a target table where each row occupies 128 bytes, with randomly generated content ensuring the actual table mem size is 128 bytes times the number of rows. The number of rows required to populate the target table is calculated based on the value of the parameter mem_object_size. To generate a potentially large target table, the script uses a cartesian join of a smaller manageable "dummy" table on itself, multiplying the record count to reach the desired size. From bb956741723061c6c98ff9b1a10ed1085a636883 Mon Sep 17 00:00:00 2001 From: Vitalii Shilin Date: Wed, 24 Jul 2024 15:38:50 +0200 Subject: [PATCH 3/5] Apply suggestions from code review Co-authored-by: allipatev <58830344+allipatev@users.noreply.github.com> --- .../creating_test_table_specified_mem_object_size.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Database-Features/creating_test_table_specified_mem_object_size.md b/Database-Features/creating_test_table_specified_mem_object_size.md index d94a739..3800edd 100644 --- a/Database-Features/creating_test_table_specified_mem_object_size.md +++ b/Database-Features/creating_test_table_specified_mem_object_size.md @@ -1,13 +1,13 @@ -## Creating a test table with a specified MEM_OBJECT_SIZE +# Creating a test table with a specified MEM_OBJECT_SIZE -This article provides a detailed guide on using the create_table_mem_object_size script to create tables with a specified memory size (MEM_OBJECT_SIZE). It might be useful for Support investigations and experiments. +This article provides a detailed guide on using the `create_table_mem_object_size` script to create tables with a specified memory size (MEM_OBJECT_SIZE). It might be useful for Support investigations and experiments. To use the script, simply create it in your desired schema. Be careful if a table named "dummy" already exists as the script will recreate it. The script is designed to create a target table where each row occupies 128 bytes, with randomly generated content ensuring the actual table mem size is 128 bytes times the number of rows. The number of rows required to populate the target table is calculated based on the value of the parameter mem_object_size. To generate a potentially large target table, the script uses a cartesian join of a smaller manageable "dummy" table on itself, multiplying the record count to reach the desired size. -# script +## script ```sql create schema test_data_gen; open schema test_data_gen; @@ -57,7 +57,7 @@ test_data_gen.dummy x2); / ``` -# Examples +## Examples 1. 100Mb ```sql From 80d62bac0db42883be8f9ed0b177792f8dd536d7 Mon Sep 17 00:00:00 2001 From: Vitalii Shilin Date: Wed, 24 Jul 2024 15:39:43 +0200 Subject: [PATCH 4/5] Update creating_test_table_specified_mem_object_size.md --- .../creating_test_table_specified_mem_object_size.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Database-Features/creating_test_table_specified_mem_object_size.md b/Database-Features/creating_test_table_specified_mem_object_size.md index 3800edd..03eea74 100644 --- a/Database-Features/creating_test_table_specified_mem_object_size.md +++ b/Database-Features/creating_test_table_specified_mem_object_size.md @@ -61,7 +61,7 @@ test_data_gen.dummy x2); 1. 100Mb ```sql -EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_100mb', 100) WITH OUTPUT; +EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_100mb', 100); --2 second on a basic singlenode vm cluster select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_100MB' and ROOT_NAME = 'TEST_DATA_GEN' @@ -70,7 +70,7 @@ select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = ' 2. 2Gb ```sql -EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_2gb', 2048) WITH OUTPUT; +EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_2gb', 2048); --40 second on a basic singlenode vm cluster select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_2GB' and ROOT_NAME = 'TEST_DATA_GEN' @@ -79,7 +79,7 @@ select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = ' 3. 10Gb ```sql -EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_10gb', 10240) WITH OUTPUT; +EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_10gb', 10240); --4 minutes on a basic singlenode vm cluster select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_10GB' and ROOT_NAME = 'TEST_DATA_GEN' From 49d35cadbe7facc6bff67385ee17144c2c827219 Mon Sep 17 00:00:00 2001 From: Vitalii Shilin Date: Thu, 25 Jul 2024 13:47:30 +0200 Subject: [PATCH 5/5] Update creating_test_table_specified_mem_object_size.md script was modified to compensate compression with additional rows --- ...ng_test_table_specified_mem_object_size.md | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/Database-Features/creating_test_table_specified_mem_object_size.md b/Database-Features/creating_test_table_specified_mem_object_size.md index 03eea74..49c6b33 100644 --- a/Database-Features/creating_test_table_specified_mem_object_size.md +++ b/Database-Features/creating_test_table_specified_mem_object_size.md @@ -48,12 +48,31 @@ SELECT RAND(1,100) AS rand15, RAND(1,100) AS rand16 - --each row takes 128 byte + --each row takes 128 byte RAW FROM (SELECT 1 from test_data_gen.dummy x1, test_data_gen.dummy x2); ]]) + + + +while true do + + res = query([[commit]]) + cur_size = query([[select mem_object_size/1024/1024 as MEM from exa_all_object_sizes where object_name = upper(']] .. table_name .. [[') and ROOT_NAME = upper(']] .. schema_name .. [[') ]]) + + if cur_size[1].MEM >= mem_object_size then + break; + end + + curr_count = query([[select table_row_count as CNT from exa_all_tables where table_name = upper(']] .. table_name .. [[') and table_schema = upper(']] .. schema_name .. [[') ]]) + local add_rows = curr_count[1].CNT * ( mem_object_size - cur_size[1].MEM) / cur_size[1].MEM + if add_rows < 100 then add_rows = 100 end + + res6 = query([[insert into ]] .. schema_name .. '.' .. table_name .. [[ select * from ]] .. schema_name .. '.' .. table_name .. [[ where rownum <= ]] .. add_rows) + output('Adding more rows to reach the target MEM_OBJECT ' .. add_rows) +end / ``` @@ -61,27 +80,27 @@ test_data_gen.dummy x2); 1. 100Mb ```sql +drop table test_data_gen.tab_100mb; EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_100mb', 100); ---2 second on a basic singlenode vm cluster - +--2 second on a 4 node Saas cluster select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_100MB' and ROOT_NAME = 'TEST_DATA_GEN' ---result 100.22835159301757812500 +--result 105.32723617553710937500 ``` 2. 2Gb ```sql +drop table test_data_gen.tab_2gb; EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_2gb', 2048); ---40 second on a basic singlenode vm cluster - +--10 second on a 4 node Saas cluster select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_2GB' and ROOT_NAME = 'TEST_DATA_GEN' ---result 2049.37433147430419921875 +--result 2048.02326202392578125000 ``` 3. 10Gb ```sql +drop table test_data_gen.TAB_10GB; EXECUTE SCRIPT create_table_mem_object_size('test_data_gen', 'tab_10gb', 10240); ---4 minutes on a basic singlenode vm cluster - +--35 seconds on a 4 node Saas cluster select mem_object_size/1024/1024 from exa_all_object_sizes where object_name = 'TAB_10GB' and ROOT_NAME = 'TEST_DATA_GEN' ---result 10240.29162216186523437500 +--result 10240.88970947265625000000 ```