diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..28a94fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.* +!.gitignore +Tapper-Benchmark-* +Debian_CPANTS.txt +nytprof* diff --git a/Changes b/Changes new file mode 100644 index 0000000..dac84dc --- /dev/null +++ b/Changes @@ -0,0 +1,63 @@ +Revision history for {{$dist->name}} + +{{$NEXT}} + +0.012 2015-09-17 + - refactor common methods between db backends + +0.011 2015-09-16 + - let the db take care of compression in raw queue + +0.010 2015-09-15 + - drop feature 'switch' - replace given/when with classic if/else + +0.009 2015-09-11 + - DB tuning/tweaking: + * avoid deadlocks due to gap locking + * fix missing finish_transaction + * be crisp about UNSIGNED and use BLOB for raw queue + * simplify resetting of AutoCommit mode + +0.008 2015-09-08 + - enable transactions for garbage collector + +0.007 2015-09-08 + - queuing mode for high-throughput incoming results + * part 1) adds raw results into separate table + * part 2) process one of those values, + likely called in bundles or from multiple workers + * part 3) garbage collect successfully processed results + +0.006 2015-08-26 + - disable caching due to wrong behaviour + +0.005 2015-08-24 + - larger field sizes + - select_benchmark_point_essentials() + Get NAME, VALUE, UNIT of a single benchmark point + - select_complete_benchmark_point() + Get all additional key/value fields of a single benchmark point + - get_single_benchmark_point() + Reconstruct a complete single benchmark point + +0.004 2015-08-20 + - get list of benchmark NAMEs + +0.003 2015-08-18 + - handling of duplicates and undefined values + +0.002 2015-08-17 + - store benchmark VALUEs as VARCHAR to avoid rounding + and allow non-number values en passant + - dependency++ to DBD::SQLite 1.48 to avoid error + 'no such column: bav2.bench_additional_value" + +0.001 2015-08-16 + - Initial release - CPAN DAY - 2015-08-16 + - based on the released code from early 2014 + - This is a library to provide a schema for + storing benchmark values in an SQL database + - Primarily targets mysql, SQLite support + is basically so far by cutting away known + trouble maker SQL statement parts + - Read more about the schema on http://benchmarkanything.org/ diff --git a/dist.ini b/dist.ini index cd4c36e..6e60cd9 100644 --- a/dist.ini +++ b/dist.ini @@ -6,3 +6,12 @@ copyright_holder = Amazon.com, Inc. or its affiliates [@TAPPER] dist = Tapper-Benchmark repository_at = github +repository = git://github.com/benchmarkanything/Tapper-Benchmark.git +repository_web = http://github.com/benchmarkanything/Tapper-Benchmark +disable_pod_coverage_tests = 0 +skip_pod_modules = qr/^Tapper::Benchmark::Query/ + +[Test::Compile] + +[Prereqs] +DBD::SQLite = 1.48 diff --git a/lib/Tapper/Benchmark.pm b/lib/Tapper/Benchmark.pm index 2d5fbb0..79593f0 100644 --- a/lib/Tapper/Benchmark.pm +++ b/lib/Tapper/Benchmark.pm @@ -1,10 +1,11 @@ package Tapper::Benchmark; +# ABSTRACT: Autonomous SQL backend to store benchmarks use strict; use warnings; my $hr_default_config = { - select_cache => 1, + select_cache => 0, default_aggregation => 'min', tables => { unit_table => 'bench_units', @@ -141,6 +142,7 @@ sub new { else { $or_self->{query} = $s_module->new({ dbh => $hr_atts->{dbh}, + driver => $hr_atts->{dbh}{Driver}{Name}, debug => $hr_atts->{debug} || 0, config => $or_self->{config}, }); @@ -238,6 +240,7 @@ sub add_single_benchmark { ADDITIONAL: for my $s_key ( keys %{$hr_point} ) { next ADDITIONAL if $s_key eq 'VALUE'; + next ADDITIONAL if not defined $hr_point->{$s_key}; # additional type my $i_addtype_id; @@ -338,6 +341,77 @@ sub add_single_benchmark { } +sub enqueue_multi_benchmark { + + my ( $or_self, $ar_data_points, $hr_options ) = @_; + + require Sereal::Encoder; + + my $s_serialized = Sereal::Encoder->new->encode($ar_data_points); + $or_self->{query}->insert_raw_bench_bundle($s_serialized); + + return 1; + +} + +# dequeues a single bundle (can contain multiple data points) +sub process_queued_multi_benchmark { + + my ( $or_self, $hr_options ) = @_; + + my $i_id; + my $s_serialized; + my $ar_data_points; + my $ar_results; + my $or_result; + my $driver = $or_self->{query}{dbh}{Driver}{Name}; + + # ===== exclusively pick single raw entry ===== + # Lock single row via processing=1 so that only one worker handles it! + $or_self->{query}{dbh}->do("set transaction isolation level read committed") if $driver eq "mysql"; # avoid deadlocks due to gap locking + $or_self->{query}->start_transaction; + eval { + $ar_results = $or_self->{query}->select_raw_bench_bundle_for_lock; + $or_result = $ar_results->fetchrow_hashref; + $i_id = $or_result->{raw_bench_bundle_id}; + if (!$i_id) { + $or_self->{query}->finish_transaction( $@ ); + $or_self->{query}{dbh}->do("set transaction isolation level repeatable read") if $driver eq "mysql"; # reset to normal gap locking + goto RETURN ; + } + $or_self->{query}->start_processing_raw_bench_bundle($i_id); + }; + $or_self->{query}->finish_transaction( $@ ); + $or_self->{query}{dbh}->do("set transaction isolation level repeatable read") if $driver eq "mysql"; # reset to normal gap locking + + # ===== process that single raw entry ===== + $or_self->{query}->start_transaction; + eval { + require Sereal::Decoder; + + $ar_results = $or_self->{query}->select_raw_bench_bundle_for_processing($i_id); + $s_serialized = $ar_results->fetchrow_hashref->{raw_bench_bundle_serialized}; + $ar_data_points = Sereal::Decoder::decode_sereal($s_serialized); + + # preserve order, otherwise add_multi_benchmark() would reorder to optimize insert + $or_self->add_multi_benchmark([$_], $hr_options) foreach @$ar_data_points; + $or_self->{query}->update_raw_bench_bundle_set_processed($i_id); + }; + $or_self->{query}->finish_transaction( $@ ); + + RETURN: + return $@ ? undef : $i_id; + +} + +# garbage collect - initially raw_bench_bundles, but also other stuff. +sub gc { + + my ( $or_self, $hr_options ) = @_; + + $or_self->{query}->delete_processed_raw_bench_bundles; +} + sub add_multi_benchmark { my ( $or_self, $ar_data_points, $hr_options ) = @_; @@ -394,6 +468,75 @@ sub search { } +sub list_benchmark_names { + + my ( $or_self, $s_pattern ) = @_; + + my $ar_pattern = defined($s_pattern) ? [$s_pattern] : []; + + my $s_key; + if ( $or_self->{cache} ) { + require JSON::XS; + $s_key = JSON::XS::encode_json($ar_pattern); + if ( my $ar_search_data = $or_self->{cache}->get("list_benchmark_names||$s_key") ) { + return $ar_search_data; + } + } + + my $ar_result = $or_self->{query} + ->select_benchmark_names( @$ar_pattern ) + ->fetchall_arrayref([0]); + my $ar_benchmark_names = [ map { $_->[0] } @$ar_result ]; + + if ( $or_self->{cache} ) { + $or_self->{cache}->set( "list_benchmark_names||$s_key" => $ar_benchmark_names ); + } + + return $ar_benchmark_names; + +} + +sub get_single_benchmark_point { + + my ( $or_self, $i_bench_value_id ) = @_; + + return {} unless $i_bench_value_id; + + # cache? + my $s_key; + if ( $or_self->{cache} ) { + require JSON::XS; + $s_key = JSON::XS::encode_json({bench_value_id => $i_bench_value_id}); + if ( my $hr_search_data = $or_self->{cache}->get("get_single_benchmark_point||$s_key") ) { + return $hr_search_data; + } + } + + # fetch all additional key/value fields + my $ar_query_result = $or_self->{query} + ->select_complete_benchmark_point( $i_bench_value_id ) + ->fetchall_arrayref({}); + + # fetch essentials, like NAME, VALUE, UNIT + my $hr_essentials = $or_self->{query} + ->select_benchmark_point_essentials( $i_bench_value_id ) + ->fetchrow_hashref(); + + # create complete BenchmarkAnything-like key/value entry + my $hr_result; + $hr_result = { map { ($_->{bench_additional_type} => $_->{bench_additional_value} ) } @$ar_query_result }; + $hr_result->{NAME} = $hr_essentials->{bench}; + $hr_result->{VALUE} = $hr_essentials->{bench_value}; + $hr_result->{UNIT} = $hr_essentials->{bench_unit} if $hr_essentials->{bench_unit}; + + # cache! + if ( $or_self->{cache} ) { + $or_self->{cache}->set( "get_single_benchmark_point||$s_key" => $hr_result ); + } + + return $hr_result; +} + sub search_array { my ( $or_self, $hr_search ) = @_; @@ -654,7 +797,7 @@ Tapper::Benchmark - Save and search benchmark points by database ], order_by => [ 'machine', - ['ASC','testrun_id',{ numeric => 1 }] + ['testrun_id','ASC',{ numeric => 1 }] ], limit => 2, offset => 1, @@ -680,7 +823,7 @@ way to the the database. A search function with complexe filters already exists. =head3 new -=over +=over 4 =item @@ -692,7 +835,7 @@ Create a new B object. config => YAML::Syck::LoadFile('~/conf/tapper_benchmark.conf'), }); -=over +=over 4 =item dbh @@ -710,9 +853,12 @@ written to STDOUT. The default is 0. =back +=back + + =head3 add_single_benchmark -=over +=over 4 =item @@ -739,33 +885,32 @@ Add one or more data points to a single benchmark to the database. force => 1 }); -=over +=over 4 -=item 1. Parameter Hash => NAME +=item 1st Parameter Hash => NAME The name of the benchmark for grouping benchmark data points. -=item 1. Parameter Hash => data +=item 1st Parameter Hash => data This parameter contains the benchmark data points. It's an array of hashes. The element C is the only required element in this hashes. The C is the benchmark data point value. -=item 1. Parameter Hash => UNIT [optional] +=item 1st Parameter Hash => UNIT [optional] Containing a unit for benchmark data point values. -=item 2. Parameter Hash => force [optional] +=item 2nd Parameter Hash => force [optional] Ignore forgivable errors while writing. =back -=head3 add_multi_benchmark +=back -=over -=item +=head3 add_multi_benchmark Add one or more data points for multiple benchmarks to the database. @@ -791,35 +936,32 @@ Add one or more data points for multiple benchmarks to the database. force => 1 }); -=over +=over 4 -=item 1. Parameter Array of Hashes => NAME +=item 1st Parameter Array of Hashes => NAME The name of the benchmark for grouping benchmark data points. -=item 1. Parameter Hash => VALUE +=item 1st Parameter Hash => VALUE The value is the benchmark data point value. -=item 1. Parameter Hash => UNIT [optional] +=item 1st Parameter Hash => UNIT [optional] Containing a unit for benchmark data point values. -=item 1. Parameter Hash => all others +=item 1st Parameter Hash => all others All other elements in the hashes are additional values added to this data point. -=item 2. Parameter Hash => force [optional] +=item 2nd Parameter Hash => force [optional] Ignore forgivable errors while writing. =back -=head3 search - -=over -=item +=head3 search Search for benchmark data points in the database. Function returns a DBI Statement Handle. @@ -843,11 +985,11 @@ Statement Handle. offset => 1, order_by => [ 'machine', - ['ASC','testrun_id'] + ['testrun_id','ASC'] ], }); -=over +=over 4 =item select [optional] @@ -973,11 +1115,8 @@ An integer value which determine the number of omitted benchmark data points. =back -=head3 search_array - -=over -=item +=head3 search_array Returning all benchmark data points as Array of Hashes. @@ -994,18 +1133,13 @@ Returning all benchmark data points as Array of Hashes. offset => 1, order_by => [ 'machine', - ['ASC','testrun_id'] + ['testrun_id','ASC'] ], }); -=back =head3 search_hash -=over - -=item - Returning all benchmark data points as Hash of Hashes. As compared to search C this function needs the parameter C. C is an Array of Strings which determine the columns used as the keys for the nested hashes. @@ -1029,15 +1163,71 @@ Every "key" create a new nested hash. offset => 1, order_by => [ 'machine', - ['ASC','testrun_id'] + ['testrun_id','ASC'] ], }); -=head3 subsume -=over +=head3 get_single_benchmark_point -=item +Get a single data points from the database including all essential +fields (NAME, VALUE, UNIT) and additional fields. + + my $point = $or_bench->get_single_benchmark_point($value_id); + + +=head3 list_benchmark_names + +Get a list of all benchmark NAMEs, optionally matching a given pattern +(SQL LIKE syntax, i.e., using C<%> as placeholder. + + $benchmarkanythingdata = $or_bench->list_benchmark_names($pattern); + +=head3 enqueue_multi_benchmark + +As a low-latency alternative to directly calling +L there is a queuing functionality. + +The C function simply writes the raw incoming +data structure serialized (and compressed) into a single row and +returns. The complementary function to this is +C which takes these values over using +the real C internally. + +=head3 process_queued_multi_benchmark + +This is part 2 of the low-latency queuing alternative to directly +calling L. + +It transactionally marks a single raw entry as being processed and +then takes over its values by calling C. It +preserves the order of entries by inserting each chunk sequentially, +to not confuse the IDs to the careful reader. After the bundle is +taken over it is marked as processed. + +This function only handles one single raw entry. It is expected to +called from co-operating multiple worker tasks or multiple times from +a wrapper. + +Currently the original raw values are B deleted immediately, just +for safety reasons, until the transactional code is death-proof (and +certified by Stuntman Mike). There is a dedicated funtion L/gc> for +that cleanup. + +The function returns the ID of the processed raw entry. + +=head3 gc + +This calls garbage collection, in particular deletes raw entries +created by C and already processed by +C. + +It is separated from those processing just for safety reasons until +the transactional code in there is waterproof. + +The gc function can cleanup more stuff in the future. + +=head3 subsume This is a maintenance function for reducing the number of data points in the database. Calling this function reduces the rows in the benchmark values table @@ -1054,7 +1244,7 @@ It is highly recommended to do this periodically for better search performance. backup => 0, }); -=over +=over 4 =item subsume_type @@ -1088,15 +1278,12 @@ isn't desired a false value must be passed. =back -=head1 Configuration - -=over -=item +=head1 Configuration The following elements are required in configuration: -=over +=over 4 =item default_aggregation @@ -1122,4 +1309,6 @@ Containing the names of the tables used bei B =item select_cache [optional] -In case of a true value the module cache some select results \ No newline at end of file +In case of a true value the module cache some select results + +=back diff --git a/lib/Tapper/Benchmark/Query.pm b/lib/Tapper/Benchmark/Query.pm index a3f99ba..f06f20f 100644 --- a/lib/Tapper/Benchmark/Query.pm +++ b/lib/Tapper/Benchmark/Query.pm @@ -1,4 +1,5 @@ package Tapper::Benchmark::Query; +# ABSTRACT: Tapper::Benchmark - querying - base class use strict; use warnings; @@ -66,6 +67,7 @@ sub start_transaction { local $or_self->{dbh}{RaiseError} = 1; eval { + $or_self->{old_AutoCommit} = $or_self->{dbh}{AutoCommit}; $or_self->{dbh}{AutoCommit} = 0; }; if ( $@ ) { @@ -99,6 +101,8 @@ sub finish_transaction { $or_self->{dbh}->commit(); } + $or_self->{dbh}{AutoCommit} = $or_self->{old_AutoCommit}; + } return 1; @@ -106,11 +110,3 @@ sub finish_transaction { } 1; - -__END__ - -=pod - -=head1 NAME - -Tapper::Benchmark::Query - Base class for the database work used by Tapper::Benchmark \ No newline at end of file diff --git a/lib/Tapper/Benchmark/Query/SQLite.pm b/lib/Tapper/Benchmark/Query/SQLite.pm new file mode 100644 index 0000000..a32baba --- /dev/null +++ b/lib/Tapper/Benchmark/Query/SQLite.pm @@ -0,0 +1,475 @@ +package Tapper::Benchmark::Query::SQLite; +# ABSTRACT: Tapper::Benchmark - querying - SQLite backend + +use strict; +use warnings; +use base 'Tapper::Benchmark::Query::common'; + +use List::MoreUtils qw( any ); + +my %h_used_selects; +my %h_default_columns = ( + 'NAME' => 'b.bench', + 'UNIT' => 'bu.bench_unit', + 'VALUE' => 'bv.bench_value', + 'VALUE_ID' => 'bv.bench_value_id', + 'CREATED' => 'bv.created_at', +); + +sub select_benchmark_values { + + my ( $or_self, $hr_search ) = @_; + + # clear selected columns + $h_used_selects{$or_self} = {}; + + # deep copy hash + require JSON::XS; + $hr_search = JSON::XS::decode_json( + JSON::XS::encode_json( $hr_search ) + ); + + my ( + $s_limit, + $s_offset, + $s_order_by, + @a_select, + @a_from, + @a_from_vals, + @a_where, + @a_where_vals, + ) = ( + q##, + q##, + q##, + ); + + # limit clause + if ( $hr_search->{limit} ) { + if ( $hr_search->{limit} =~ /^\d+$/ ) { + $s_limit = "LIMIT $hr_search->{limit}"; + } + else { + require Carp; + Carp::confess("invalid limit value '$hr_search->{limit}'"); + return; + } + } + + # offset clause + if ( $hr_search->{offset} ) { + if ( $hr_search->{offset} =~ /^\d+$/ ) { + $s_offset = "OFFSET $hr_search->{offset}"; + } + else { + require Carp; + Carp::confess("invalid offset value '$hr_search->{offset}'"); + return; + } + } + + # where clause + my $i_counter = 0; + if ( $hr_search->{where} ) { + + for my $ar_where ( @{$hr_search->{where}} ) { + if ( any { $ar_where->[1] eq $_ } keys %h_default_columns ) { + my $s_column = splice( @{$ar_where}, 1, 1 ); + push @a_where, $or_self->create_where_clause( $h_default_columns{$s_column}, $ar_where ); + push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; + } + else { + my $s_additional_type = splice( @{$ar_where}, 1, 1 ); + my $hr_additional_type = $or_self + ->select_addtype_by_name( $s_additional_type ) + ->fetchrow_hashref() + ; + if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { + require Carp; + Carp::confess("benchmark additional value '$s_additional_type' not exists"); + return; + } + push @a_from, " + JOIN ( + $or_self->{config}{tables}{additional_relation_table} bar$i_counter + JOIN $or_self->{config}{tables}{additional_value_table} bav$i_counter + ON ( bav$i_counter.bench_additional_value_id = bar$i_counter.bench_additional_value_id ) + ) + ON ( + bar$i_counter.bench_value_id = bv.bench_value_id + AND bav$i_counter.bench_additional_type_id = ? + ) + "; + push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; + push @a_where, $or_self->create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); + push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; + $i_counter++; + } + } + } + + # select clause + my $b_aggregate_all = 0; + if ( $hr_search->{select} ) { + for my $i_counter ( 0..$#{$hr_search->{select}} ) { + if ( ref $hr_search->{select}[$i_counter] ne 'ARRAY' ) { + $hr_search->{select}[$i_counter] = ['',$hr_search->{select}[$i_counter]]; + } + elsif ( !$b_aggregate_all && $hr_search->{select}[$i_counter][0] ne q## ) { + $b_aggregate_all = 1; + for my $s_clause (qw/ order_by limit offset /) { + if ( $hr_search->{$s_clause} ) { + require Carp; + Carp::confess("cannot use '$s_clause' with aggregation"); + } + } + } + } + } + push @{$hr_search->{select} ||= []}, map {['',$_]} keys %h_default_columns; + + for my $ar_select ( @{$hr_search->{select}} ) { + + my ( $s_column, $s_select ) = $or_self->create_select_column( + $ar_select, $i_counter, $b_aggregate_all, + ); + + if ( $s_select ) { + + push @a_select, $s_select; + + if ( $s_column ) { + + my $hr_additional_type = $or_self + ->select_addtype_by_name( $s_column ) + ->fetchrow_hashref() + ; + if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { + require Carp; + Carp::confess("benchmark additional value '$s_column' not exists"); + return; + } + + push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; + push @a_from, " + LEFT JOIN ( + $or_self->{config}{tables}{additional_relation_table} bar$i_counter + JOIN $or_self->{config}{tables}{additional_value_table} bav$i_counter + ON ( bav$i_counter.bench_additional_value_id = bar$i_counter.bench_additional_value_id ) + ) + ON ( + bar$i_counter.bench_value_id = bv.bench_value_id + AND bav$i_counter.bench_additional_type_id = ? + ) + "; + $i_counter++; + } + } + + } + + # order_by clause + if ( $hr_search->{order_by} ) { + my @a_order_by_possible = keys %h_default_columns; + my @a_order_by_direction = qw/ ASC DESC /; + if ( $hr_search->{select} ) { + push @a_order_by_possible, map { $_->[1] } @{$hr_search->{select}}; + } + my @a_order_by; + for my $order_column ( @{$hr_search->{order_by}} ) { + if ( ref $order_column ) { + if ( any { $order_column->[0] eq $_ } @a_order_by_possible ) { + if ( any { $order_column->[1] eq $_ } @a_order_by_direction ) { + my $s_numeric_cast = q##; + if ( $order_column->[2] && $order_column->[2]{numeric} ) { + $s_numeric_cast = '0 + '; + } + if ( any { $order_column->[0] eq $_ } keys %h_default_columns ) { + push @a_order_by, "$s_numeric_cast$h_default_columns{$order_column->[0]} $order_column->[1]"; + } + else { + push @a_order_by, "$s_numeric_cast$order_column->[0] $order_column->[1]"; + } + } + else { + require Carp; + Carp::confess("unknown order by direction '$order_column->[1]'"); + return; + } + } + else { + require Carp; + Carp::confess("unknown order by column '$order_column->[0]'"); + return; + } + } + else { + if ( any { $order_column eq $_ } @a_order_by_possible ) { + if ( any { $order_column eq $_ } keys %h_default_columns ) { + push @a_order_by, "$h_default_columns{$order_column} ASC"; + } + else { + push @a_order_by, "$order_column ASC"; + } + } + else { + require Carp; + Carp::confess("unknown order by column '$order_column'"); + return; + } + } + } + $s_order_by = 'ORDER BY ' . (join ', ', @a_order_by) + } + + # replace placeholders inside of raw sql where clause + my $s_raw_where = $hr_search->{where_sql}; + if ( $s_raw_where ) { + $s_raw_where =~ s/ + \${(.+?)} + / + $h_used_selects{$or_self}{$1} + ? $h_used_selects{$or_self}{$1} + : die "column '$1' not exists in SELECT clause" + /gex; + } + + return $or_self->execute_query( + " + SELECT + " . ( join ",\n", map {"$_"} @a_select ) . " + FROM + $or_self->{config}{tables}{benchmark_table} b + JOIN $or_self->{config}{tables}{benchmark_value_table} bv + ON ( bv.bench_id = b.bench_id ) + LEFT JOIN $or_self->{config}{tables}{unit_table} bu + ON ( bu.bench_unit_id = b.bench_unit_id ) + " . ( join "\n", @a_from ) . " + WHERE + b.active = 1 + AND bv.active = 1 + " . + ( @a_where ? join "\n", map { "AND $_" } @a_where : q## ) . + ( $s_raw_where ? " $s_raw_where" : q## ) . + " + $s_order_by + $s_limit + $s_offset + ", + @a_from_vals, + @a_where_vals, + ); + +} + +sub create_select_column { + + my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; + + my $s_aggr_func = q##; + my ( $s_aggr, $s_column ) = @{$ar_select}; + my $s_return_select = q##; + + AGGR: { + if ( $s_aggr eq q## ) { + # aggregate all columns if a single column is aggregated + if ( $b_aggregate_all ) { + $s_aggr = $or_self->{config}{default_aggregation}; + redo AGGR; + } + $s_return_select = '${COLUMN}'; + } + elsif ( $s_aggr eq 'min' ) { + $s_return_select = 'MIN( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'max' ) { + $s_return_select = 'MAX( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'avg' ) { + $s_return_select = 'AVG( ${COLUMN} )'; + } + # Geometric Mean, unsupported in SQLite due to lack of EXP(), + # see http://stackoverflow.com/questions/13190064/how-to-find-power-of-a-number-in-sqlite + # + # elsif ( $s_aggr eq 'gem' ) { + # $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; + # } + elsif ( $s_aggr eq 'sum' ) { + $s_return_select = 'SUM( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnt' ) { + $s_return_select = 'COUNT( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnd' ) { + $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; + } + else { + require Carp; + Carp::confess("unknown aggregate function '$s_aggr'"); + return; + } + } # AGGR + + my ( $s_return_column ); + my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; + + if ( $h_used_selects{$or_self}{$s_replace_as} ) { + return; + } + if ( any { $s_column eq $_ } keys %h_default_columns ) { + $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; + } + else { + $s_return_column = $s_column; + $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; + } + + $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; + + return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); + +} + +sub select_raw_bench_bundle_for_lock { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT raw_bench_bundle_id + FROM raw_bench_bundles + WHERE processed=0 AND + processing=0 + ORDER BY raw_bench_bundle_id ASC + LIMIT 1 + ", @a_vals ); +} + +sub select_raw_bench_bundle_for_processing { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT raw_bench_bundle_serialized + FROM raw_bench_bundles + WHERE raw_bench_bundle_id = ? + LIMIT 1 + ", @a_vals ); +} + +sub insert_addtyperelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT OR IGNORE INTO $or_self->{config}{tables}{additional_type_relation_table} + ( bench_id, bench_additional_type_id, created_at ) + VALUES + ( ?, ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_unit { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{unit_table} + ( bench_unit, created_at ) + VALUES + ( ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_benchmark { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{benchmark_table} + ( bench, bench_unit_id, active, created_at ) + VALUES + ( ?, ?, 1, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_benchmark_value { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT OR IGNORE INTO $or_self->{config}{tables}{benchmark_value_table} + ( bench_id, bench_subsume_type_id, bench_value, active, created_at ) + VALUES + ( ?, ?, ?, 1, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub copy_additional_values { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_relation_table} + ( bench_value_id, bench_additional_value_id, active, created_at ) + SELECT + ?, bench_additional_value_id, 1, CURRENT_TIMESTAMP + FROM + $or_self->{config}{tables}{additional_relation_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub insert_addtype { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_type_table} + ( bench_additional_type, created_at ) + VALUES + ( ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_addvalue { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{additional_value_table} + ( bench_additional_type_id, bench_additional_value, created_at ) + VALUES + ( ?, ?, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +sub insert_addvaluerelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT OR IGNORE INTO $or_self->{config}{tables}{additional_relation_table} + ( bench_value_id, bench_additional_value_id, active, created_at ) + VALUES + ( ?, ?, 1, CURRENT_TIMESTAMP ) + ", @a_vals ); + +} + +1; diff --git a/lib/Tapper/Benchmark/Query/common.pm b/lib/Tapper/Benchmark/Query/common.pm new file mode 100644 index 0000000..42df21f --- /dev/null +++ b/lib/Tapper/Benchmark/Query/common.pm @@ -0,0 +1,531 @@ +package Tapper::Benchmark::Query::common; +# ABSTRACT: Tapper::Benchmark - querying - backend base class + +use strict; +use warnings; +use base 'Tapper::Benchmark::Query'; + +use List::MoreUtils qw( any ); + +my %h_used_selects; +my %h_default_columns = ( + 'NAME' => 'b.bench', + 'UNIT' => 'bu.bench_unit', + 'VALUE' => 'bv.bench_value', + 'VALUE_ID' => 'bv.bench_value_id', + 'CREATED' => 'bv.created_at', +); + +sub default_columns { + return %h_default_columns; +} + +sub benchmark_operators { + return ( '=', '!=', 'like', 'not like', '<', '>', '<=', '>=' ); +} + +sub create_where_clause { + + my ( $or_self, $s_column_name, $ar_value ) = @_; + + my $s_where_clause = q##; + if ( $ar_value->[0] eq 'not like' ) { + $s_where_clause = "$s_column_name NOT LIKE ?"; + } + elsif ( $ar_value->[0] eq 'like' ) { + $s_where_clause = "$s_column_name LIKE ?"; + } + elsif ( + $ar_value->[0] eq '<' + || $ar_value->[0] eq '>' + || $ar_value->[0] eq '<=' + || $ar_value->[0] eq '>=' + ) { + $s_where_clause = "$s_column_name $ar_value->[0] ?"; + } + elsif ( $ar_value->[0] eq '=' ) { + if ( $#{$ar_value} > 1 ) { + $s_where_clause = "$s_column_name IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; + } + else { + $s_where_clause = "$s_column_name = ?"; + } + } + elsif ( $ar_value->[0] eq '!=' ) { + if ( $#{$ar_value} > 1 ) { + $s_where_clause = "$s_column_name NOT IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; + } + else { + $s_where_clause = "$s_column_name != ?"; + } + } + else { + require Carp; + Carp::confess("unknown operator '$ar_value->[0]'"); + return; + } + + return $s_where_clause; + +} + +sub create_period_check { + + my ( $or_self, $s_column, $dt_from, $dt_to ) = @_; + + my @a_vals; + my $s_where; + if ( $dt_from ) { + if ( my ( $s_date, $s_time ) = $dt_from =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { + $s_where .= "\nAND $s_column > ?"; + push @a_vals, $s_date . ( $s_time || ' 00:00:00' ); + } + else { + require Carp; + Carp::confess(q#unknown date format for 'date_from'#); + return; + } + } + if ( $dt_to ) { + if ( my ( $s_date, $s_time ) = $dt_to =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { + $s_where .= "\nAND $s_column < ?"; + push @a_vals, $s_date . ( $s_time || ' 23:59:59' ); + } + else { + require Carp; + Carp::confess(q#unknown date format for 'date_to'#); + return; + } + } + + return { + vals => \@a_vals, + where => $s_where, + }; + +} + +sub select_benchmark_point_essentials { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + b.bench, + bv.bench_value, + bu.bench_unit + FROM + $or_self->{config}{tables}{benchmark_table} b + JOIN + $or_self->{config}{tables}{benchmark_value_table} bv + ON + b.bench_id = bv.bench_id + LEFT JOIN + $or_self->{config}{tables}{unit_table} bu + ON + b.bench_unit_id = bu.bench_unit_id + WHERE + bv.bench_value_id = ? + ; + ", @a_vals ); + +} + +sub select_complete_benchmark_point { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT + bat.bench_additional_type, + bav.bench_additional_value + FROM + benchs b + JOIN + bench_values bv + ON + b.bench_id = bv.bench_id + JOIN + bench_additional_type_relations batr + ON + bv.bench_id = batr.bench_id + JOIN + bench_additional_types bat + ON + batr.bench_additional_type_id = bat.bench_additional_type_id + JOIN + bench_additional_relations bar + ON + bv.bench_value_id = bar.bench_value_id + JOIN + bench_additional_values bav + ON + bar.bench_additional_value_id = bav.bench_additional_value_id AND + bat.bench_additional_type_id = bav.bench_additional_type_id + WHERE + bv.bench_value_id = ? + ORDER BY + bat.bench_additional_type"; + return $or_self->execute_query( $query, @a_vals ); +} + +sub select_addtype_by_name { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_type_id + FROM $or_self->{config}{tables}{additional_type_table} + WHERE bench_additional_type = ? + ", @a_vals ); + +} + +sub select_min_subsume_type { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_subsume_type_id + FROM $or_self->{config}{tables}{subsume_type_table} + ORDER BY bench_subsume_type_rank ASC + LIMIT 1 + " ); + +} + +sub select_subsume_type { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT + bench_subsume_type_id, + bench_subsume_type_rank, + datetime_strftime_pattern + FROM + $or_self->{config}{tables}{subsume_type_table} + WHERE + bench_subsume_type = ? + ", @a_vals ); + +} + +sub select_check_subsumed_values { + + my ( $or_self, $hr_vals ) = @_; + + if (! $hr_vals->{subsume_type_id} ) { + require Carp; + Carp::confess(q#required parameter 'subsume_type_id' is missing#); + return; + } + + my $hr_period_check = $or_self->create_period_check( + 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} + ); + + return $or_self->execute_query( + " + SELECT + bv.bench_value_id + FROM + bench_values bv + JOIN bench_subsume_types bet + ON ( bv.bench_subsume_type_id = bet.bench_subsume_type_id ) + WHERE + bet.bench_subsume_type_rank > ( + SELECT beti.bench_subsume_type_rank + FROM bench_subsume_types beti + WHERE bench_subsume_type_id = ? + ) + $hr_period_check->{where} + LIMIT + 1 + ", + $hr_vals->{subsume_type_id}, + @{$hr_period_check->{vals}}, + ); + +} + +sub select_data_values_for_subsume { + + my ( $or_self, $hr_vals ) = @_; + + my $hr_period_check = $or_self->create_period_check( + 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} + ); + + my @a_addexclude_vals; + my $s_addexclude_where = q##; + if ( $hr_vals->{exclude_additionals} && @{$hr_vals->{exclude_additionals}} ) { + $s_addexclude_where = 'AND bav.bench_additional_type_id NOT IN (' . (join ',', map {'?'} @{$hr_vals->{exclude_additionals}}) . ')'; + push @a_addexclude_vals, @{$hr_vals->{exclude_additionals}}; + } + + return $or_self->execute_query( + " + SELECT + b.bench_id, + bv.bench_value_id, + bv.created_at, + bv.bench_value, + bet.bench_subsume_type_rank, + GROUP_CONCAT( + bav.bench_additional_type_id, + '|', + bav.bench_additional_value_id + ORDER BY + bav.bench_additional_type_id, + bav.bench_additional_value_id + SEPARATOR + '-' + ) AS additionals + FROM + benchs b + JOIN bench_values bv + ON ( bv.bench_id = b.bench_id ) + JOIN bench_subsume_types bet + ON ( bet.bench_subsume_type_id = bv.bench_subsume_type_id ) + LEFT JOIN ( + bench_additional_relations bar + JOIN bench_additional_values bav + ON ( bav.bench_additional_value_id = bar.bench_additional_value_id ) + ) + ON ( + bar.active = 1 + AND bar.bench_value_id = bv.bench_value_id + $s_addexclude_where + ) + WHERE + b.active = 1 + AND bv.active = 1 + $hr_period_check->{where} + GROUP BY + bet.bench_subsume_type_rank, + b.bench_id, + bv.created_at, + bv.bench_value, + bv.bench_value_id + ORDER BY + b.bench_id, + additionals, + bv.created_at + ", + @a_addexclude_vals, + @{$hr_period_check->{vals}}, + ); +} + +sub select_benchmark { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_id + FROM $or_self->{config}{tables}{benchmark_table} + WHERE bench = ? + ", @a_vals ); + +} + +sub select_benchmark_names { + + my ( $or_self, @a_vals ) = @_; + + my $query = " + SELECT DISTINCT bench + FROM $or_self->{config}{tables}{benchmark_table}"; + $query .= " + WHERE bench LIKE ? " if @a_vals; + return $or_self->execute_query( $query, @a_vals ); + +} + +sub select_unit { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_unit_id + FROM $or_self->{config}{tables}{unit_table} + WHERE bench_unit = ? + ", @a_vals ); + +} + +sub select_addtype { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_type_id + FROM $or_self->{config}{tables}{additional_type_table} + WHERE bench_additional_type = ? + ", @a_vals ); + +} + +sub select_addvalue { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_additional_value_id + FROM $or_self->{config}{tables}{additional_value_table} + WHERE bench_additional_type_id = ? AND bench_additional_value = ? + ", @a_vals ); + +} + +sub select_addtyperelation { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + SELECT bench_id, bench_additional_type_id, created_at + FROM $or_self->{config}{tables}{additional_type_relation_table} + WHERE bench_id = ? AND bench_additional_type_id = ? + ", @a_vals ); + +} + +sub insert_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + INSERT INTO raw_bench_bundles + (raw_bench_bundle_serialized) + VALUES ( ? ) + ", @a_vals ); + +} + +sub copy_benchmark_backup_value { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{benchmark_backup_value_table} + ( bench_value_id, bench_id, bench_subsume_type_id, bench_value, active, created_at ) + SELECT + ?, bench_id, bench_subsume_type_id, bench_value, active, created_at + FROM + $or_self->{config}{tables}{benchmark_value_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub copy_benchmark_backup_additional_relations { + + my ( $or_self, $hr_vals ) = @_; + + for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { + if (! $hr_vals->{$s_param} ) { + require Carp; + Carp::confess("missing parameter '$s_param'"); + return; + } + } + + return $or_self->execute_query( " + INSERT INTO $or_self->{config}{tables}{backup_additional_relation_table} + ( bench_backup_value_id, bench_additional_value_id, active, created_at ) + SELECT + ?, bench_additional_value_id, active, created_at + FROM + $or_self->{config}{tables}{additional_relation_table} + WHERE + bench_value_id = ? + ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); + +} + +sub update_benchmark_backup_value { + + my ( $or_self, $hr_vals ) = @_; + + return $or_self->execute_query( " + UPDATE $or_self->{config}{tables}{benchmark_backup_value_table} + SET bench_value_id = ? + WHERE bench_value_id = ? + ", @{$hr_vals}{qw/ + new_bench_value_id + old_bench_value_id + /} ); + +} + +sub start_processing_raw_bench_bundle { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processing = 1 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + +sub update_raw_bench_bundle_set_processed { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + UPDATE raw_bench_bundles + SET processed=1, + processing=0 + WHERE raw_bench_bundle_id = ? + ", @a_vals ); + +} + +sub delete_benchmark_additional_relations { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{additional_relation_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + +sub delete_benchmark_value { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM $or_self->{config}{tables}{benchmark_value_table} + WHERE bench_value_id = ? + ", @a_vals ); + +} + +# Garbage Collection +sub delete_processed_raw_bench_bundles { + + my ( $or_self, @a_vals ) = @_; + + return $or_self->execute_query( " + DELETE FROM raw_bench_bundles + WHERE processed=1 AND + processing=0 + ", @a_vals ); + +} + +1; diff --git a/lib/Tapper/Benchmark/Query/mysql.pm b/lib/Tapper/Benchmark/Query/mysql.pm index c00c690..00f3704 100644 --- a/lib/Tapper/Benchmark/Query/mysql.pm +++ b/lib/Tapper/Benchmark/Query/mysql.pm @@ -1,9 +1,9 @@ package Tapper::Benchmark::Query::mysql; +# ABSTRACT: Tapper::Benchmark - querying - MySQL backend use strict; use warnings; -use feature 'switch'; -use base 'Tapper::Benchmark::Query'; +use base 'Tapper::Benchmark::Query::common'; use List::MoreUtils qw( any ); @@ -16,162 +16,6 @@ my %h_default_columns = ( 'CREATED' => 'bv.created_at', ); -sub default_columns { - return %h_default_columns; -} - -sub benchmark_operators { - return ( '=', '!=', 'like', 'not like', '<', '>', '<=', '>=' ); -} - -sub create_where_clause { - - my ( $s_column_name, $ar_value ) = @_; - - my $s_where_clause = q##; - if ( $ar_value->[0] eq 'not like' ) { - $s_where_clause = "$s_column_name NOT LIKE ?"; - } - elsif ( $ar_value->[0] eq 'like' ) { - $s_where_clause = "$s_column_name LIKE ?"; - } - elsif ( - $ar_value->[0] eq '<' - || $ar_value->[0] eq '>' - || $ar_value->[0] eq '<=' - || $ar_value->[0] eq '>=' - ) { - $s_where_clause = "$s_column_name $ar_value->[0] ?"; - } - elsif ( $ar_value->[0] eq '=' ) { - if ( $#{$ar_value} > 1 ) { - $s_where_clause = "$s_column_name IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; - } - else { - $s_where_clause = "$s_column_name = ?"; - } - } - elsif ( $ar_value->[0] eq '!=' ) { - if ( $#{$ar_value} > 1 ) { - $s_where_clause = "$s_column_name NOT IN (" . (join ',', map {'?'} 2..@{$ar_value}) . ')'; - } - else { - $s_where_clause = "$s_column_name != ?"; - } - } - else { - require Carp; - Carp::confess("unknown operator '$ar_value->[0]'"); - return; - } - - return $s_where_clause; - -} - -sub create_select_column { - - my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; - - my $s_aggr_func = q##; - my ( $s_aggr, $s_column ) = @{$ar_select}; - my $s_return_select = q##; - - AGGR: { - given( $s_aggr ) { - when ( q## ) { - # aggregate all columns if a single column is aggregated - if ( $b_aggregate_all ) { - $s_aggr = $or_self->{config}{default_aggregation}; - redo AGGR; - } - $s_return_select = '${COLUMN}'; - } - when ( 'min' ) { - $s_return_select = 'MIN( ${COLUMN} )'; - } - when ( 'max' ) { - $s_return_select = 'MAX( ${COLUMN} )'; - } - when ( 'avg' ) { - $s_return_select = 'AVG( ${COLUMN} )'; - } - when ( 'gem' ) { - $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; - } - when ( 'sum' ) { - $s_return_select = 'SUM( ${COLUMN} )'; - } - when ( 'cnt' ) { - $s_return_select = 'COUNT( ${COLUMN} )'; - } - when ( 'cnd' ) { - $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; - } - default { - require Carp; - Carp::confess("unknown aggregate function '$s_aggr'"); - return; - } - } - } # AGGR - - my ( $s_return_column ); - my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; - - if ( $h_used_selects{$or_self}{$s_replace_as} ) { - return; - } - if ( any { $s_column eq $_ } keys %h_default_columns ) { - $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; - } - else { - $s_return_column = $s_column; - $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; - } - - $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - - return ( $s_return_column, "$s_return_select AS $s_replace_as", ); - -} - -sub create_period_check { - - my ( $s_column, $dt_from, $dt_to ) = @_; - - my @a_vals; - my $s_where; - if ( $dt_from ) { - if ( my ( $s_date, $s_time ) = $dt_from =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { - $s_where .= "\nAND $s_column > ?"; - push @a_vals, $s_date . ( $s_time || ' 00:00:00' ); - } - else { - require Carp; - Carp::confess(q#unknown date format for 'date_from'#); - return; - } - } - if ( $dt_to ) { - if ( my ( $s_date, $s_time ) = $dt_to =~ /(\d{4}-\d{2}-\d{2})( \d{2}:\d{2}:\d{2})?/ ) { - $s_where .= "\nAND $s_column < ?"; - push @a_vals, $s_date . ( $s_time || ' 23:59:59' ); - } - else { - require Carp; - Carp::confess(q#unknown date format for 'date_to'#); - return; - } - } - - return { - vals => \@a_vals, - where => $s_where, - }; - -} - sub select_benchmark_values { my ( $or_self, $hr_search ) = @_; @@ -231,17 +75,18 @@ sub select_benchmark_values { for my $ar_where ( @{$hr_search->{where}} ) { if ( any { $ar_where->[1] eq $_ } keys %h_default_columns ) { my $s_column = splice( @{$ar_where}, 1, 1 ); - push @a_where, create_where_clause( $h_default_columns{$s_column}, $ar_where ); + push @a_where, $or_self->create_where_clause( $h_default_columns{$s_column}, $ar_where ); push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; } else { + my $s_additional_type = splice( @{$ar_where}, 1, 1 ); my $hr_additional_type = $or_self - ->select_addtype_by_name( splice( @{$ar_where}, 1, 1 ) ) + ->select_addtype_by_name( $s_additional_type ) ->fetchrow_hashref() ; if ( !$hr_additional_type || !$hr_additional_type->{bench_additional_type_id} ) { require Carp; - Carp::confess("benchmark additional value '$ar_where->[1]' not exists"); + Carp::confess("benchmark additional value '$s_additional_type' not exists"); return; } push @a_from, " @@ -256,7 +101,7 @@ sub select_benchmark_values { ) "; push @a_from_vals, $hr_additional_type->{bench_additional_type_id}; - push @a_where, create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); + push @a_where, $or_self->create_where_clause( "bav$i_counter.bench_additional_value", $ar_where ); push @a_where_vals , @{$ar_where}[1..$#{$ar_where}]; $i_counter++; } @@ -417,213 +262,98 @@ sub select_benchmark_values { } -sub select_addtype_by_name { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_additional_type_id - FROM $or_self->{config}{tables}{additional_type_table} - WHERE bench_additional_type = ? - ", @a_vals ); - -} - -sub select_min_subsume_type { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_subsume_type_id - FROM $or_self->{config}{tables}{subsume_type_table} - ORDER BY bench_subsume_type_rank ASC - LIMIT 1 - " ); - -} - -sub select_subsume_type { - - my ( $or_self, @a_vals ) = @_; +sub create_select_column { - return $or_self->execute_query( " - SELECT - bench_subsume_type_id, - bench_subsume_type_rank, - datetime_strftime_pattern - FROM - $or_self->{config}{tables}{subsume_type_table} - WHERE - bench_subsume_type = ? - ", @a_vals ); + my ( $or_self, $ar_select, $i_counter, $b_aggregate_all ) = @_; -} + warn ("***** mysql - create_select_column"); + my $s_aggr_func = q##; + my ( $s_aggr, $s_column ) = @{$ar_select}; + my $s_return_select = q##; -sub select_check_subsumed_values { + AGGR: { + if ( $s_aggr eq q## ) { + # aggregate all columns if a single column is aggregated + if ( $b_aggregate_all ) { + $s_aggr = $or_self->{config}{default_aggregation}; + redo AGGR; + } + $s_return_select = '${COLUMN}'; + } + elsif ( $s_aggr eq 'min' ) { + $s_return_select = 'MIN( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'max' ) { + $s_return_select = 'MAX( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'avg' ) { + $s_return_select = 'AVG( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'gem' ) { + $s_return_select = 'EXP( SUM( LOG( ${COLUMN} ) ) / COUNT( ${COLUMN} ) )'; + } + elsif ( $s_aggr eq 'sum' ) { + $s_return_select = 'SUM( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnt' ) { + $s_return_select = 'COUNT( ${COLUMN} )'; + } + elsif ( $s_aggr eq 'cnd' ) { + $s_return_select = 'COUNT( DISTINCT ${COLUMN} )'; + } + else { + require Carp; + Carp::confess("unknown aggregate function '$s_aggr'"); + return; + } + } # AGGR - my ( $or_self, $hr_vals ) = @_; + my ( $s_return_column ); + my $s_replace_as = $s_aggr ? $s_aggr . "_$s_column" : $s_column; - if (! $hr_vals->{subsume_type_id} ) { - require Carp; - Carp::confess(q#required parameter 'subsume_type_id' is missing#); + if ( $h_used_selects{$or_self}{$s_replace_as} ) { return; } - - my $hr_period_check = create_period_check( - 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} - ); - - return $or_self->execute_query( - " - SELECT - bv.bench_value_id - FROM - bench_values bv - JOIN bench_subsume_types bet - ON ( bv.bench_subsume_type_id = bet.bench_subsume_type_id ) - WHERE - bet.bench_subsume_type_rank > ( - SELECT beti.bench_subsume_type_rank - FROM bench_subsume_types beti - WHERE bench_subsume_type_id = ? - ) - $hr_period_check->{where} - LIMIT - 1 - ", - $hr_vals->{subsume_type_id}, - @{$hr_period_check->{vals}}, - ); - -} - -sub select_data_values_for_subsume { - - my ( $or_self, $hr_vals ) = @_; - - my $hr_period_check = create_period_check( - 'bv.created_at', $hr_vals->{date_from}, $hr_vals->{date_to} - ); - - my @a_addexclude_vals; - my $s_addexclude_where = q##; - if ( $hr_vals->{exclude_additionals} && @{$hr_vals->{exclude_additionals}} ) { - $s_addexclude_where = 'AND bav.bench_additional_type_id NOT IN (' . (join ',', map {'?'} @{$hr_vals->{exclude_additionals}}) . ')'; - push @a_addexclude_vals, @{$hr_vals->{exclude_additionals}}; + if ( any { $s_column eq $_ } keys %h_default_columns ) { + $h_used_selects{$or_self}{$s_replace_as} = $h_default_columns{$s_column}; + } + else { + $s_return_column = $s_column; + $h_used_selects{$or_self}{$s_replace_as} = "bav$i_counter.bench_additional_value"; } - return $or_self->execute_query( - " - SELECT - b.bench_id, - bv.bench_value_id, - bv.created_at, - bv.bench_value, - bet.bench_subsume_type_rank, - GROUP_CONCAT( - bav.bench_additional_type_id, - '|', - bav.bench_additional_value_id - ORDER BY - bav.bench_additional_type_id, - bav.bench_additional_value_id - SEPARATOR - '-' - ) AS additionals - FROM - benchs b - JOIN bench_values bv - ON ( bv.bench_id = b.bench_id ) - JOIN bench_subsume_types bet - ON ( bet.bench_subsume_type_id = bv.bench_subsume_type_id ) - LEFT JOIN ( - bench_additional_relations bar - JOIN bench_additional_values bav - ON ( bav.bench_additional_value_id = bar.bench_additional_value_id ) - ) - ON ( - bar.active = 1 - AND bar.bench_value_id = bv.bench_value_id - $s_addexclude_where - ) - WHERE - b.active = 1 - AND bv.active = 1 - $hr_period_check->{where} - GROUP BY - bet.bench_subsume_type_rank, - b.bench_id, - bv.created_at, - bv.bench_value, - bv.bench_value_id - ORDER BY - b.bench_id, - additionals, - bv.created_at - ", - @a_addexclude_vals, - @{$hr_period_check->{vals}}, - ); -} - -sub select_benchmark { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_id - FROM $or_self->{config}{tables}{benchmark_table} - WHERE bench = ? - ", @a_vals ); - -} - -sub select_unit { - - my ( $or_self, @a_vals ) = @_; + $s_return_select =~ s/\${COLUMN}/$h_used_selects{$or_self}{$s_replace_as}/g; - return $or_self->execute_query( " - SELECT bench_unit_id - FROM $or_self->{config}{tables}{unit_table} - WHERE bench_unit = ? - ", @a_vals ); + return ( $s_return_column, "$s_return_select AS '$s_replace_as'", ); } -sub select_addtype { +sub select_raw_bench_bundle_for_lock { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - SELECT bench_additional_type_id - FROM $or_self->{config}{tables}{additional_type_table} - WHERE bench_additional_type = ? + SELECT raw_bench_bundle_id + FROM raw_bench_bundles + WHERE processed=0 AND + processing=0 + ORDER BY raw_bench_bundle_id ASC + LIMIT 1 + FOR UPDATE ", @a_vals ); - } -sub select_addvalue { +sub select_raw_bench_bundle_for_processing { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - SELECT bench_additional_value_id - FROM $or_self->{config}{tables}{additional_value_table} - WHERE bench_additional_type_id = ? AND bench_additional_value = ? - ", @a_vals ); - -} - -sub select_addtyperelation { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - SELECT bench_id, bench_additional_type_id, created_at - FROM $or_self->{config}{tables}{additional_type_relation_table} - WHERE bench_id = ? AND bench_additional_type_id = ? + SELECT raw_bench_bundle_serialized + FROM raw_bench_bundles + WHERE raw_bench_bundle_id = ? + LIMIT 1 + FOR UPDATE ", @a_vals ); - } sub insert_addtyperelation { @@ -631,7 +361,7 @@ sub insert_addtyperelation { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_type_relation_table} + INSERT IGNORE INTO $or_self->{config}{tables}{additional_type_relation_table} ( bench_id, bench_additional_type_id, created_at ) VALUES ( ?, ?, NOW() ) @@ -648,6 +378,8 @@ sub insert_unit { ( bench_unit, created_at ) VALUES ( ?, NOW() ) + ON DUPLICATE KEY + UPDATE bench_unit_id=LAST_INSERT_ID(bench_unit_id) ", @a_vals ); } @@ -661,6 +393,8 @@ sub insert_benchmark { ( bench, bench_unit_id, active, created_at ) VALUES ( ?, ?, 1, NOW() ) + ON DUPLICATE KEY + UPDATE bench_id=LAST_INSERT_ID(bench_id) ", @a_vals ); } @@ -670,7 +404,7 @@ sub insert_benchmark_value { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{benchmark_value_table} + INSERT IGNORE INTO $or_self->{config}{tables}{benchmark_value_table} ( bench_id, bench_subsume_type_id, bench_value, active, created_at ) VALUES ( ?, ?, ?, 1, NOW() ) @@ -703,65 +437,17 @@ sub copy_additional_values { } -sub copy_benchmark_backup_value { - - my ( $or_self, $hr_vals ) = @_; - - for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { - if (! $hr_vals->{$s_param} ) { - require Carp; - Carp::confess("missing parameter '$s_param'"); - return; - } - } - - return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{benchmark_backup_value_table} - ( bench_value_id, bench_id, bench_subsume_type_id, bench_value, active, created_at ) - SELECT - ?, bench_id, bench_subsume_type_id, bench_value, active, created_at - FROM - $or_self->{config}{tables}{benchmark_value_table} - WHERE - bench_value_id = ? - ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); - -} - -sub copy_benchmark_backup_additional_relations { - - my ( $or_self, $hr_vals ) = @_; - - for my $s_param (qw/ new_bench_value_id old_bench_value_id /) { - if (! $hr_vals->{$s_param} ) { - require Carp; - Carp::confess("missing parameter '$s_param'"); - return; - } - } - - return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{backup_additional_relation_table} - ( bench_backup_value_id, bench_additional_value_id, active, created_at ) - SELECT - ?, bench_additional_value_id, active, created_at - FROM - $or_self->{config}{tables}{additional_relation_table} - WHERE - bench_value_id = ? - ", @{$hr_vals}{qw/ new_bench_value_id old_bench_value_id /} ); - -} - sub insert_addtype { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_type_table} + INSERT IGNORE INTO $or_self->{config}{tables}{additional_type_table} ( bench_additional_type, created_at ) VALUES ( ?, NOW() ) + ON DUPLICATE KEY + UPDATE bench_additional_type_id=LAST_INSERT_ID(bench_additional_type_id) ", @a_vals ); } @@ -775,6 +461,8 @@ sub insert_addvalue { ( bench_additional_type_id, bench_additional_value, created_at ) VALUES ( ?, ?, NOW() ) + ON DUPLICATE KEY + UPDATE bench_additional_value_id=LAST_INSERT_ID(bench_additional_value_id) ", @a_vals ); } @@ -784,7 +472,7 @@ sub insert_addvaluerelation { my ( $or_self, @a_vals ) = @_; return $or_self->execute_query( " - INSERT INTO $or_self->{config}{tables}{additional_relation_table} + INSERT IGNORE INTO $or_self->{config}{tables}{additional_relation_table} ( bench_value_id, bench_additional_value_id, active, created_at ) VALUES ( ?, ?, 1, NOW() ) @@ -792,49 +480,4 @@ sub insert_addvaluerelation { } -sub update_benchmark_backup_value { - - my ( $or_self, $hr_vals ) = @_; - - return $or_self->execute_query( " - UPDATE $or_self->{config}{tables}{benchmark_backup_value_table} - SET bench_value_id = ? - WHERE bench_value_id = ? - ", @{$hr_vals}{qw/ - new_bench_value_id - old_bench_value_id - /} ); - -} - -sub delete_benchmark_additional_relations { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{additional_relation_table} - WHERE bench_value_id = ? - ", @a_vals ); - -} - -sub delete_benchmark_value { - - my ( $or_self, @a_vals ) = @_; - - return $or_self->execute_query( " - DELETE FROM $or_self->{config}{tables}{benchmark_value_table} - WHERE bench_value_id = ? - ", @a_vals ); - -} - 1; - -__END__ - -=pod - -=head1 NAME - -Tapper::Benchmark::Query::mysql - Base class for the database work used by Tapper::Benchmark when MySQL is used \ No newline at end of file diff --git a/share/tapper-benchmark-create-schema.SQLite b/share/tapper-benchmark-create-schema.SQLite new file mode 100644 index 0000000..4f78bde --- /dev/null +++ b/share/tapper-benchmark-create-schema.SQLite @@ -0,0 +1,181 @@ +DROP TABLE IF EXISTS `bench_backup_additional_relations`; +DROP TABLE IF EXISTS `bench_backup_values`; +DROP TABLE IF EXISTS `bench_additional_relations`; +DROP TABLE IF EXISTS `bench_additional_type_relations`; +DROP TABLE IF EXISTS `bench_additional_values`; +DROP TABLE IF EXISTS `bench_additional_types`; +DROP TABLE IF EXISTS `bench_values`; +DROP TABLE IF EXISTS `benchs`; +DROP TABLE IF EXISTS `bench_units`; +DROP TABLE IF EXISTS `bench_subsume_types`; +DROP TABLE IF EXISTS `raw_bench_bundles`; + +CREATE TABLE `bench_subsume_types` ( + `bench_subsume_type_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_subsume_type` VARCHAR(32) NOT NULL , -- COMMENT 'unique string identifier', + `bench_subsume_type_rank` TINYINT(3) NOT NULL , -- COMMENT 'subsume type order', + `datetime_strftime_pattern` VARCHAR(32) NULL , -- COMMENT 'format pattern for per DateTime->strftime for grouping', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' +) ; -- COMMENT 'types of subsume values'; + +CREATE TABLE `bench_units` ( + `bench_unit_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_unit` VARCHAR(767) NOT NULL , -- COMMENT 'unique string identifier', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date', +) ; -- COMMENT 'units for benchmark data points'; + +CREATE TABLE `benchs` ( + `bench_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_unit_id` TINYINT(3) NULL, + `bench` VARCHAR(767) NOT NULL , -- COMMENT 'unique string identifier', + `active` TINYINT(3) NOT NULL , -- COMMENT 'is entry still active (1=yes,0=no)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_benchs_01` + FOREIGN KEY (`bench_unit_id`) + REFERENCES `bench_units` (`bench_unit_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'containing benchmark head data'; + +CREATE TABLE `bench_values` ( + `bench_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_id` INT(10) NOT NULL , -- COMMENT 'FK to benchs', + `bench_subsume_type_id` TINYINT(3) NOT NULL , -- COMMENT 'FK to bench_subsume_types', + `bench_value` VARCHAR(767) NULL , -- COMMENT 'value for bench data point', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_bench_values_01` + FOREIGN KEY (`bench_id`) + REFERENCES `benchs` (`bench_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_values_02` + FOREIGN KEY (`bench_subsume_type_id`) + REFERENCES `bench_subsume_types` (`bench_subsume_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'containing data points for benchmark'; + +CREATE TABLE `bench_additional_types` ( + `bench_additional_type_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_additional_type` VARCHAR(767) NOT NULL , -- COMMENT 'unique string identifier', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' +) ; -- COMMENT 'types of additional values for benchmark data points'; + +CREATE TABLE `bench_additional_values` ( + `bench_additional_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_additional_type_id` SMALLINT(5) NOT NULL , -- COMMENT 'FK to bench_additional_types', + `bench_additional_value` VARCHAR(767) NOT NULL , -- COMMENT 'additional value', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_bench_additional_values_01` + FOREIGN KEY (`bench_additional_type_id`) + REFERENCES `bench_additional_types` (`bench_additional_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'additional values for benchmark data point'; + +CREATE TABLE `bench_additional_type_relations` ( + `bench_id` INT(10) NOT NULL , -- COMMENT 'FK to benchs (PK)', + `bench_additional_type_id` SMALLINT(5) NOT NULL , -- COMMENT 'FK to bench_additional_types (PK)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + PRIMARY KEY (`bench_id`,`bench_additional_type_id`), + CONSTRAINT `fk_bench_additional_type_relations_01` + FOREIGN KEY (`bench_id`) + REFERENCES `benchs` (`bench_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_additional_type_relations_02` + FOREIGN KEY (`bench_additional_type_id`) + REFERENCES `bench_additional_types` (`bench_additional_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'additional values for benchmark data point'; + +CREATE TABLE `bench_additional_relations` ( + `bench_value_id` INT(10) NOT NULL , -- COMMENT 'FK to bench_values', + `bench_additional_value_id` INT(10) NOT NULL , -- COMMENT 'FK to bench_additional_values', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + PRIMARY KEY (`bench_value_id`,`bench_additional_value_id`), + CONSTRAINT `fk_bench_additional_relations_01` + FOREIGN KEY (`bench_value_id`) + REFERENCES `bench_values` (`bench_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_additional_relations_02` + FOREIGN KEY (`bench_additional_value_id`) + REFERENCES `bench_additional_values` (`bench_additional_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT 'add additional values to benchmarks'; + +CREATE TABLE `bench_backup_values` ( + `bench_backup_value_id` INTEGER PRIMARY KEY NOT NULL , -- COMMENT 'unique key (PK)', + `bench_value_id` int(10) NOT NULL , -- COMMENT 'FK to bench_values', + `bench_id` int(10) NOT NULL , -- COMMENT 'FK to benchs', + `bench_subsume_type_id` tinyint(3) NOT NULL , -- COMMENT 'FK to bench_subsume_types', + `bench_value` float DEFAULT NULL , -- COMMENT 'value for bench data point', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + CONSTRAINT `fk_bench_backup_values_01` + FOREIGN KEY (`bench_id`) + REFERENCES `benchs` (`bench_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_backup_values_02` + FOREIGN KEY (`bench_subsume_type_id`) + REFERENCES `bench_subsume_types` (`bench_subsume_type_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_backup_values_03` + FOREIGN KEY (`bench_value_id`) + REFERENCES `bench_values` (`bench_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT='backup table for data points for benchmark'; + +CREATE TABLE `bench_backup_additional_relations` ( + `bench_backup_value_id` int(10) NOT NULL , -- COMMENT 'FK to bench_backup_values', + `bench_additional_value_id` int(10) NOT NULL , -- COMMENT 'FK to bench_additional_values', + `active` tinyint(3) NOT NULL , -- COMMENT 'is entry still active (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP , -- COMMENT 'creation date', + PRIMARY KEY (`bench_backup_value_id`,`bench_additional_value_id`), + CONSTRAINT `fk_bench_backup_additional_relations_01` + FOREIGN KEY (`bench_backup_value_id`) + REFERENCES `bench_backup_values` (`bench_backup_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION, + CONSTRAINT `fk_bench_backup_additional_relations_02` + FOREIGN KEY (`bench_additional_value_id`) + REFERENCES `bench_additional_values` (`bench_additional_value_id`) + ON DELETE NO ACTION + ON UPDATE NO ACTION +) ; -- COMMENT='add additional values to benchmarks'; + +CREATE TABLE `raw_bench_bundles` ( + `raw_bench_bundle_id` INT(10) PRIMARY KEY NOT NULL, + `raw_bench_bundle_serialized` BLOB NOT NULL, + `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', + `processed` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP -- COMMENT 'creation date' +) ; -- COMMENT='queue raw incoming data for later processing'; +-- Usage: +-- * create elements: +-- INSERT INTO raw_bench_bundles (raw_bench_bundle_serialized) VALUES ('hot content'); +-- * get next: +-- SELECT raw_bench_bundle_id, raw_bench_bundle_serialized FROM raw_bench_bundles WHERE processed=0 ORDER BY raw_bench_bundle_id ASC LIMIT 1; +-- * mark as done: +-- UPDATE raw_bench_bundles SET processed=1 WHERE raw_bench_bundle_id=1; + +INSERT INTO bench_subsume_types + ( bench_subsume_type, bench_subsume_type_rank, datetime_strftime_pattern, created_at ) +VALUES + ( 'atomic' , 1, NULL , datetime() ), + ( 'second' , 2, '%Y%m%d%H%M%S' , datetime() ), + ( 'minute' , 3, '%Y%m%d%H%M' , datetime() ), + ( 'hour' , 4, '%Y%m%d%H' , datetime() ), + ( 'day' , 5, '%Y%m%d' , datetime() ), + ( 'week' , 6, '%Y%W' , datetime() ), + ( 'month' , 7, '%Y%m' , datetime() ), + ( 'year' , 8, '%Y' , datetime() ) +; \ No newline at end of file diff --git a/sql/benchmark.sql b/share/tapper-benchmark-create-schema.mysql similarity index 87% rename from sql/benchmark.sql rename to share/tapper-benchmark-create-schema.mysql index 7c3c09c..84a18fd 100644 --- a/sql/benchmark.sql +++ b/share/tapper-benchmark-create-schema.mysql @@ -12,6 +12,7 @@ DROP TABLE IF EXISTS `bench_values`; DROP TABLE IF EXISTS `benchs`; DROP TABLE IF EXISTS `bench_units`; DROP TABLE IF EXISTS `bench_subsume_types`; +DROP TABLE IF EXISTS `raw_bench_bundles`; CREATE TABLE `bench_subsume_types` ( `bench_subsume_type_id` TINYINT(3) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', @@ -25,7 +26,7 @@ CREATE TABLE `bench_subsume_types` ( CREATE TABLE `bench_units` ( `bench_unit_id` TINYINT(3) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', - `bench_unit` VARCHAR(64) NOT NULL COMMENT 'unique string identifier', + `bench_unit` VARCHAR(767) NOT NULL COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_unit_id`), UNIQUE INDEX `ux_bench_units_01` (`bench_unit` ASC) @@ -34,7 +35,7 @@ CREATE TABLE `bench_units` ( CREATE TABLE `benchs` ( `bench_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_unit_id` TINYINT(3) UNSIGNED NULL, - `bench` VARCHAR(64) NOT NULL COMMENT 'unique string identifier', + `bench` VARCHAR(767) NOT NULL COMMENT 'unique string identifier', `active` TINYINT(3) UNSIGNED NOT NULL COMMENT 'is entry still active (1=yes,0=no)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_id`), @@ -45,13 +46,13 @@ CREATE TABLE `benchs` ( REFERENCES `bench_units` (`bench_unit_id`) ON DELETE NO ACTION ON UPDATE NO ACTION -) COMMENT 'containg benchmark head data'; +) COMMENT 'containing benchmark head data'; CREATE TABLE `bench_values` ( `bench_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_id` INT(10) UNSIGNED NOT NULL COMMENT 'FK to benchs', `bench_subsume_type_id` TINYINT(3) UNSIGNED NOT NULL COMMENT 'FK to bench_subsume_types', - `bench_value` FLOAT NULL COMMENT 'value for bench data point', + `bench_value` VARCHAR(767) NULL COMMENT 'value for bench data point', `active` tinyint(3) unsigned NOT NULL COMMENT 'is entry still active (0=no,1=yes)', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_value_id`), @@ -71,7 +72,7 @@ CREATE TABLE `bench_values` ( CREATE TABLE `bench_additional_types` ( `bench_additional_type_id` SMALLINT(5) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', - `bench_additional_type` VARCHAR(32) NOT NULL COMMENT 'unique string identifier', + `bench_additional_type` VARCHAR(767) NOT NULL COMMENT 'unique string identifier', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_additional_type_id`), UNIQUE INDEX `ux_bench_additional_types_01` (`bench_additional_type` ASC) @@ -80,7 +81,7 @@ CREATE TABLE `bench_additional_types` ( CREATE TABLE `bench_additional_values` ( `bench_additional_value_id` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT COMMENT 'unique key (PK)', `bench_additional_type_id` SMALLINT(5) UNSIGNED NOT NULL COMMENT 'FK to bench_additional_types', - `bench_additional_value` VARCHAR(128) NOT NULL COMMENT 'additional value', + `bench_additional_value` VARCHAR(767) NOT NULL COMMENT 'additional value', `created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date', PRIMARY KEY (`bench_additional_value_id`), INDEX `fk_bench_additional_values_01` (`bench_additional_type_id` ASC), @@ -180,6 +181,21 @@ CREATE TABLE `bench_backup_additional_relations` ( ON UPDATE NO ACTION ) COMMENT='add additional values to benchmarks'; +CREATE TABLE `raw_bench_bundles` ( + `raw_bench_bundle_id` INT(10) UNSIGNED PRIMARY KEY AUTO_INCREMENT NOT NULL, + `raw_bench_bundle_serialized` LONGBLOB NOT NULL, + `processing` tinyint(3) NOT NULL DEFAULT 0 , -- COMMENT 'is entry processed (0=no,1=yes)', + `processed` tinyint(3) NOT NULL DEFAULT 0 COMMENT 'is entry processed (0=no,1=yes)', + `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'creation date' +) ROW_FORMAT=COMPRESSED COMMENT='queue raw incoming data for later processing'; +-- Usage: +-- * create elements: +-- INSERT INTO raw_bench_bundles (raw_bench_bundle_serialized) VALUES ('hot content'); +-- * get next: +-- SELECT raw_bench_bundle_id, raw_bench_bundle_serialized FROM raw_bench_bundles WHERE processed=0 ORDER BY raw_bench_bundle_id ASC LIMIT 1; +-- * mark as done: +-- UPDATE raw_bench_bundles SET processed=1 WHERE raw_bench_bundle_id=1; + SET FOREIGN_KEY_CHECKS = 1; INSERT INTO bench_subsume_types