From 1eda9b8e673b656a0d7336bf068a962aecb08950 Mon Sep 17 00:00:00 2001 From: mingodad Date: Wed, 11 Jul 2018 10:16:08 +0200 Subject: [PATCH 1/7] Code refactoring duplicated code --- src/fann.c | 144 ++++++++++++++++++++++------------------------------- 1 file changed, 59 insertions(+), 85 deletions(-) diff --git a/src/fann.c b/src/fann.c index 9af7e388..0f2cc231 100644 --- a/src/fann.c +++ b/src/fann.c @@ -29,39 +29,48 @@ /* #define FANN_NO_SEED */ -FANN_EXTERNAL struct fann *FANN_API fann_create_standard(unsigned int num_layers, ...) +static unsigned int *getUIntArray(unsigned int n, va_list va) { - struct fann *ann; - va_list layer_sizes; - int i; - int status; - int arg; - unsigned int *layers = (unsigned int *) calloc(num_layers, sizeof(unsigned int)); + unsigned int i; + unsigned int arg; + unsigned int arg_sum; + unsigned int *uint_array = (unsigned int *) calloc(n, sizeof(unsigned int)); - if(layers == NULL) + if(uint_array == NULL) { fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); return NULL; } - va_start(layer_sizes, num_layers); - - status = 1; - for(i = 0; i < (int) num_layers; i++) + arg_sum = 0; + for(i = 0; i < n; i++) { - arg = va_arg(layer_sizes, unsigned int); - if(arg < 0 || arg > 1000000) - status = 0; - layers[i] = arg; + arg = va_arg(va, unsigned int); + arg_sum += arg; + if(arg_sum > 1000000) + { + fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); + free(uint_array); + return NULL; + } + uint_array[i] = arg; } - va_end(layer_sizes); + return uint_array; +} - if(!status) +FANN_EXTERNAL struct fann *FANN_API fann_create_standard(unsigned int num_layers, ...) +{ + struct fann *ann; + va_list layer_sizes; + unsigned int *layers; + + va_start(layer_sizes, num_layers); + layers = getUIntArray(num_layers, layer_sizes); + if(layers == NULL) { - fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); - free(layers); return NULL; } + va_end(layer_sizes); ann = fann_create_standard_array(num_layers, layers); @@ -70,45 +79,28 @@ FANN_EXTERNAL struct fann *FANN_API fann_create_standard(unsigned int num_layers return ann; } -FANN_EXTERNAL struct fann *FANN_API fann_create_standard_array(unsigned int num_layers, +FANN_EXTERNAL struct fann *FANN_API fann_create_standard_array(unsigned int num_layers, const unsigned int *layers) { - return fann_create_sparse_array(1, num_layers, layers); + return fann_create_sparse_array(1, num_layers, layers); } -FANN_EXTERNAL struct fann *FANN_API fann_create_sparse(float connection_rate, +FANN_EXTERNAL struct fann *FANN_API fann_create_sparse(float connection_rate, unsigned int num_layers, ...) { struct fann *ann; va_list layer_sizes; - int i; - int status; - int arg; - unsigned int *layers = (unsigned int *) calloc(num_layers, sizeof(unsigned int)); + unsigned int *layers; + va_start(layer_sizes, num_layers); + layers = getUIntArray(num_layers, layer_sizes); if(layers == NULL) { - fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); return NULL; } - va_start(layer_sizes, num_layers); - status = 1; - for(i = 0; i < (int) num_layers; i++) - { - arg = va_arg(layer_sizes, unsigned int); - if(arg < 0 || arg > 1000000) - status = 0; - layers[i] = arg; - } va_end(layer_sizes); - if(!status) - { - fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); - free(layers); - return NULL; - } ann = fann_create_sparse_array(connection_rate, num_layers, layers); free(layers); @@ -389,35 +381,17 @@ FANN_EXTERNAL struct fann *FANN_API fann_create_sparse_array(float connection_ra FANN_EXTERNAL struct fann *FANN_API fann_create_shortcut(unsigned int num_layers, ...) { struct fann *ann; - int i; - int status; - int arg; va_list layer_sizes; - unsigned int *layers = (unsigned int *) calloc(num_layers, sizeof(unsigned int)); + unsigned int *layers; + va_start(layer_sizes, num_layers); + layers = getUIntArray(num_layers, layer_sizes); if(layers == NULL) { - fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); return NULL; } - - va_start(layer_sizes, num_layers); - status = 1; - for(i = 0; i < (int) num_layers; i++) - { - arg = va_arg(layer_sizes, unsigned int); - if(arg < 0 || arg > 1000000) - status = 0; - layers[i] = arg; - } va_end(layer_sizes); - if(!status) - { - fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM); - free(layers); - return NULL; - } ann = fann_create_shortcut_array(num_layers, layers); @@ -580,7 +554,7 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) fann_type last_steepness = 0; unsigned int last_activation_function = 0; #else - fann_type max_sum = 0; + fann_type max_sum = 0; #endif /* first set the input */ @@ -783,13 +757,13 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) last_activation_function = activation_function; #else neuron_sum = fann_mult(steepness, neuron_sum); - + max_sum = 150/steepness; if(neuron_sum > max_sum) neuron_sum = max_sum; else if(neuron_sum < -max_sum) neuron_sum = -max_sum; - + neuron_it->sum = neuron_sum; fann_activation_switch(activation_function, neuron_sum, neuron_it->value); @@ -826,7 +800,7 @@ FANN_EXTERNAL void FANN_API fann_destroy(struct fann *ann) fann_safe_free(ann->cascade_activation_functions); fann_safe_free(ann->cascade_activation_steepnesses); fann_safe_free(ann->cascade_candidate_scores); - + #ifndef FIXEDFANN fann_safe_free( ann->scale_mean_in ); fann_safe_free( ann->scale_deviation_in ); @@ -838,7 +812,7 @@ FANN_EXTERNAL void FANN_API fann_destroy(struct fann *ann) fann_safe_free( ann->scale_new_min_out ); fann_safe_free( ann->scale_factor_out ); #endif - + fann_safe_free(ann); } @@ -1306,10 +1280,10 @@ FANN_EXTERNAL void FANN_API fann_print_parameters(struct fann *ann) for(i = 0; i < ann->cascade_activation_steepnesses_count; i++) printf("Cascade activation steepnesses[%d] :%8.3f\n", i, ann->cascade_activation_steepnesses[i]); - + printf("Cascade candidate groups :%4d\n", ann->cascade_num_candidate_groups); printf("Cascade no. of candidates :%4d\n", fann_get_cascade_num_candidates(ann)); - + /* TODO: dump scale parameters */ #endif } @@ -1420,7 +1394,7 @@ FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct f source_index = 0; destination_index = 0; - + /* The following assumes that the last unused bias has no connections */ /* for each layer */ @@ -1616,8 +1590,8 @@ struct fann *fann_allocate_structure(unsigned int num_layers) ann->scale_deviation_out = NULL; ann->scale_new_min_out = NULL; ann->scale_factor_out = NULL; -#endif - +#endif + /* variables used for cascade correlation (reasonable defaults) */ ann->cascade_output_change_fraction = 0.01f; ann->cascade_candidate_change_fraction = 0.01f; @@ -1632,8 +1606,8 @@ struct fann *fann_allocate_structure(unsigned int num_layers) ann->cascade_min_cand_epochs = 50; ann->cascade_candidate_scores = NULL; ann->cascade_activation_functions_count = 10; - ann->cascade_activation_functions = - (enum fann_activationfunc_enum *)calloc(ann->cascade_activation_functions_count, + ann->cascade_activation_functions = + (enum fann_activationfunc_enum *)calloc(ann->cascade_activation_functions_count, sizeof(enum fann_activationfunc_enum)); if(ann->cascade_activation_functions == NULL) { @@ -1641,7 +1615,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers) free(ann); return NULL; } - + ann->cascade_activation_functions[0] = FANN_SIGMOID; ann->cascade_activation_functions[1] = FANN_SIGMOID_SYMMETRIC; ann->cascade_activation_functions[2] = FANN_GAUSSIAN; @@ -1654,8 +1628,8 @@ struct fann *fann_allocate_structure(unsigned int num_layers) ann->cascade_activation_functions[9] = FANN_COS; ann->cascade_activation_steepnesses_count = 4; - ann->cascade_activation_steepnesses = - (fann_type *)calloc(ann->cascade_activation_steepnesses_count, + ann->cascade_activation_steepnesses = + (fann_type *)calloc(ann->cascade_activation_steepnesses_count, sizeof(fann_type)); if(ann->cascade_activation_steepnesses == NULL) { @@ -1664,7 +1638,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers) free(ann); return NULL; } - + ann->cascade_activation_steepnesses[0] = (fann_type)0.25; ann->cascade_activation_steepnesses[1] = (fann_type)0.5; ann->cascade_activation_steepnesses[2] = (fann_type)0.75; @@ -1680,14 +1654,14 @@ struct fann *fann_allocate_structure(unsigned int num_layers) ann->rprop_delta_min = 0.0; ann->rprop_delta_max = 50.0; ann->rprop_delta_zero = 0.1f; - + /* Variables for use with SARPROP training (reasonable defaults) */ ann->sarprop_weight_decay_shift = -6.644f; ann->sarprop_step_error_threshold_factor = 0.1f; ann->sarprop_step_error_shift = 1.385f; ann->sarprop_temperature = 0.015f; ann->sarprop_epoch = 0; - + fann_init_error_data((struct fann_error *) ann); #ifdef FIXEDFANN @@ -1743,7 +1717,7 @@ int fann_allocate_scale(struct fann *ann) SCALE_ALLOCATE( scale_new_min, out, -1.0 ) SCALE_ALLOCATE( scale_factor, out, 1.0 ) #undef SCALE_ALLOCATE -#endif +#endif return 0; } @@ -1845,13 +1819,13 @@ void fann_seed_rand() } else { - if(fread(&foo, sizeof(foo), 1, fp) != 1) + if(fread(&foo, sizeof(foo), 1, fp) != 1) { gettimeofday(&t, NULL); foo = t.tv_usec; #ifdef DEBUG printf("unable to read from /dev/urandom\n"); -#endif +#endif } fclose(fp); } From 5e926218a8869a4c74f7b9f77763b1365732d2dd Mon Sep 17 00:00:00 2001 From: mingodad Date: Thu, 12 Jul 2018 10:46:30 +0200 Subject: [PATCH 2/7] Refactoring and removing duplicated code, add a function for timming --- src/fann.c | 73 ++++++++++++++++++++++------------------------ src/include/fann.h | 1 + 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/fann.c b/src/fann.c index 0f2cc231..92f1d9ed 100644 --- a/src/fann.c +++ b/src/fann.c @@ -1381,7 +1381,8 @@ FANN_EXTERNAL void FANN_API fann_get_bias_array(struct fann *ann, unsigned int * } } -FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct fann_connection *connections) +static void fann_apply_to_weights(struct fann *ann, struct fann_connection *connections, + unsigned int from_neuron, unsigned int to_neuron, fann_type weight, int isSet) { struct fann_neuron *first_neuron; struct fann_layer *layer_it; @@ -1395,20 +1396,30 @@ FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct f source_index = 0; destination_index = 0; - /* The following assumes that the last unused bias has no connections */ - /* for each layer */ for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++){ /* for each neuron */ for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++){ /* for each connection */ for (idx = neuron_it->first_con; idx < neuron_it->last_con; idx++){ - /* Assign the source, destination and weight */ - connections->from_neuron = (unsigned int)(ann->connections[source_index] - first_neuron); - connections->to_neuron = destination_index; - connections->weight = ann->weights[source_index]; + if(isSet) + { + /* If the source and destination neurons match, assign the weight */ + if (((int)from_neuron == ann->connections[source_index] - first_neuron) && + (to_neuron == destination_index)) + { + ann->weights[source_index] = weight; + } + } + else + { + /* Assign the source, destination and weight */ + connections->from_neuron = (unsigned int)(ann->connections[source_index] - first_neuron); + connections->to_neuron = destination_index; + connections->weight = ann->weights[source_index]; - connections++; + connections++; + } source_index++; } destination_index++; @@ -1416,6 +1427,12 @@ FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct f } } +FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct fann_connection *connections) +{ + /* The following assumes that the last unused bias has no connections */ + fann_apply_to_weights(ann, connections, 0, 0, 0, 0); +} + FANN_EXTERNAL void FANN_API fann_set_weight_array(struct fann *ann, struct fann_connection *connections, unsigned int num_connections) { @@ -1430,39 +1447,10 @@ FANN_EXTERNAL void FANN_API fann_set_weight_array(struct fann *ann, FANN_EXTERNAL void FANN_API fann_set_weight(struct fann *ann, unsigned int from_neuron, unsigned int to_neuron, fann_type weight) { - struct fann_neuron *first_neuron; - struct fann_layer *layer_it; - struct fann_neuron *neuron_it; - unsigned int idx; - unsigned int source_index; - unsigned int destination_index; - - first_neuron = ann->first_layer->first_neuron; - - source_index = 0; - destination_index = 0; - /* Find the connection, simple brute force search through the network for one or more connections that match to minimize datastructure dependencies. Nothing is done if the connection does not already exist in the network. */ - - /* for each layer */ - for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++){ - /* for each neuron */ - for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++){ - /* for each connection */ - for (idx = neuron_it->first_con; idx < neuron_it->last_con; idx++){ - /* If the source and destination neurons match, assign the weight */ - if (((int)from_neuron == ann->connections[source_index] - first_neuron) && - (to_neuron == destination_index)) - { - ann->weights[source_index] = weight; - } - source_index++; - } - destination_index++; - } - } + fann_apply_to_weights(ann, NULL, from_neuron, to_neuron, weight, 1); } FANN_EXTERNAL void FANN_API fann_get_weights(struct fann *ann, fann_type *weights) @@ -1840,3 +1828,12 @@ void fann_seed_rand() #endif } +long long fann_mstime(void) { + struct timeval tv; + long long ust; + + gettimeofday(&tv, NULL); + ust = ((long long)tv.tv_sec)*1000000; + ust += tv.tv_usec; + return ust/1000; +} diff --git a/src/include/fann.h b/src/include/fann.h index 5adda119..ebfbdc72 100644 --- a/src/include/fann.h +++ b/src/include/fann.h @@ -615,6 +615,7 @@ FANN_EXTERNAL void FANN_API fann_disable_seed_rand(); */ FANN_EXTERNAL void FANN_API fann_enable_seed_rand(); +FANN_EXTERNAL long FANN_API fann_mstime(void); #ifdef FIXEDFANN From 735ca95e3569c7a3bf8c91105610051ddda1a7dc Mon Sep 17 00:00:00 2001 From: mingodad Date: Thu, 12 Jul 2018 11:10:11 +0200 Subject: [PATCH 3/7] Add a benchmark example program --- examples/Makefile | 8 ++- examples/nn-benchmark.c | 144 ++++++++++++++++++++++++++++++++++++++ examples/parallel_train.c | 9 +-- 3 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 examples/nn-benchmark.c diff --git a/examples/Makefile b/examples/Makefile index b3d9b0f3..b99c2ec4 100644 --- a/examples/Makefile +++ b/examples/Makefile @@ -2,9 +2,13 @@ GCC=gcc CFLAGS=-I ../src/include -LDFLAGS=-L ../src/ +#LDFLAGS=-L ../src/ +LDFLAGS=-L ../build/src/ + +TARGETS = xor_train xor_test xor_test_fixed simple_train \ + steepness_train simple_test robot mushroom cascade_train \ + scaling_test scaling_train nn-benchmark parallel_train -TARGETS = xor_train xor_test xor_test_fixed simple_train steepness_train simple_test robot mushroom cascade_train scaling_test scaling_train DEBUG_TARGETS = xor_train_debug xor_test_debug xor_test_fixed_debug cascade_train_debug all: $(TARGETS) diff --git a/examples/nn-benchmark.c b/examples/nn-benchmark.c new file mode 100644 index 00000000..6d3c0c41 --- /dev/null +++ b/examples/nn-benchmark.c @@ -0,0 +1,144 @@ +/* This benchmark generates a random set of inputs and outputs + * on a large network and tests the backpropagation speed in order + * for the net to arrive at a certain level of error. */ + +#include +#include +#include + +#include "fann.h" +#include "parallel_fann.h" + +//#define ONLY_TEST + +#define NUM_INPUTS 60 //120 //300 +#define NUM_OUTPUTS 10 //40 //100 + +int math_random(int low, int up) { + fann_type r = rand() * (1.0 / (RAND_MAX + 1.0)); + r *= (up - low) + 1.0; + return (int)r+low; +} + +void gen_dataset(fann_type **inputs, fann_type **outputs, int setsize) { + *inputs = calloc(1, sizeof(fann_type)*setsize*NUM_INPUTS); + *outputs = calloc(1, sizeof(fann_type)*setsize*NUM_OUTPUTS); + int ilen = NUM_INPUTS; + int olen = NUM_OUTPUTS; + int olen_1 = olen - 1; + + fann_type *in = *inputs; + fann_type *out = *outputs; + for (int j = 0; j < setsize; j++) { + for (int k = 0; k < ilen; k++) in[k] = rand() & 1; + //int r = rand() & olen_1; + int r = math_random(0, olen_1); + out[r] = 1; + //printf("%d : %d\n", j, r); + //for (int k = 0; k < olen; k++) { + // out[k] = (k == r) ? 1 : 0; + //} + in+= ilen; + out+= olen; + } +} + +int main(int argc, char *argv[]) { + const char fn_net[] = "benchmark_float.net"; + int setsize = 1000; + unsigned int i = 0, ilen; + unsigned int num_threads = 1; + if(argc == 2) + num_threads = atoi(argv[1]); +#ifndef ONLY_TEST + fann_type *inputs, *outputs; + gen_dataset(&inputs, &outputs, setsize); +#endif + fann_type *test_inputs, *test_outputs; + gen_dataset(&test_inputs, &test_outputs, setsize); + + struct fann_train_data *test_data; + +#ifndef ONLY_TEST + const unsigned int num_layers = 3; + const unsigned int num_neurons_hidden = NUM_INPUTS*2; + const float desired_error = (const float) 0.0001; + float desired_error_reached; + const unsigned int max_epochs = 3000; + const unsigned int epochs_between_reports = 10; + struct fann *ann; + struct fann_train_data *train_data; + + printf("Creating network.\n"); + + train_data = fann_create_train_array(setsize, NUM_INPUTS, inputs, NUM_OUTPUTS, outputs); + + ann = fann_create_standard(num_layers, + train_data->num_input, num_neurons_hidden, train_data->num_output); + + printf("Training network.\n"); + + fann_set_activation_function_hidden(ann, FANN_SIGMOID_SYMMETRIC); + fann_set_activation_function_output(ann, FANN_SIGMOID); + //fann_set_training_algorithm(ann, FANN_TRAIN_INCREMENTAL); + fann_set_training_algorithm(ann, FANN_TRAIN_RPROP); + fann_set_learning_rate(ann, 0.5f); + fann_randomize_weights(ann, -2.0f, 2.0f); + + /*fann_set_training_algorithm(ann, FANN_TRAIN_INCREMENTAL); */ + + //fann_train_on_data(ann, train_data, max_epochs, epochs_between_reports, desired_error); + + + long before = fann_mstime(); + for(i = 1; i <= max_epochs; i++) + { + long start = fann_mstime(); + double error = (num_threads > 1) + ? fann_train_epoch_irpropm_parallel(ann, train_data, num_threads) + : fann_train_epoch(ann, train_data); + long elapsed = fann_mstime() - start; + printf("Epochs %8d. Current error: %.10f :: %ld\n", i, error, elapsed); + desired_error_reached = fann_desired_error_reached(ann, desired_error); + if(desired_error_reached == 0) + break; + } + printf("Time spent %ld ms\n", fann_mstime()-before); + +#else + struct fann *ann = fann_create_from_file(fn_net); +#endif + + test_data = fann_create_train_array(setsize, NUM_INPUTS, test_inputs, NUM_OUTPUTS, test_outputs); + ilen = fann_length_train_data(test_data); + printf("Testing network. %d\n", ilen); + + + fann_reset_MSE(ann); + for(i = 0; i < ilen; i++) + { + fann_test(ann, test_data->input[i], test_data->output[i]); + } + + printf("MSE error on test data: %f\n", fann_get_MSE(ann)); + +#ifndef ONLY_TEST + printf("Saving network.\n"); + + fann_save(ann, fn_net); +#endif + + printf("Cleaning up.\n"); + fann_destroy(ann); + +#ifndef ONLY_TEST + fann_destroy_train(train_data); + free(inputs); + free(outputs); +#endif + fann_destroy_train(test_data); + free(test_inputs); + free(test_outputs); + + return 0; +} diff --git a/examples/parallel_train.c b/examples/parallel_train.c index 9f8804ba..730e6868 100644 --- a/examples/parallel_train.c +++ b/examples/parallel_train.c @@ -26,7 +26,7 @@ int main(int argc, const char* argv[]) unsigned int num_threads = 1; struct fann_train_data *data; struct fann *ann; - long before; + long before, totaltime; float error; unsigned int i; @@ -39,13 +39,14 @@ int main(int argc, const char* argv[]) fann_set_activation_function_hidden(ann, FANN_SIGMOID_SYMMETRIC); fann_set_activation_function_output(ann, FANN_SIGMOID); - before = GetTickCount(); + before = fann_mstime(); for(i = 1; i <= max_epochs; i++) { error = num_threads > 1 ? fann_train_epoch_irpropm_parallel(ann, data, num_threads) : fann_train_epoch(ann, data); - printf("Epochs %8d. Current error: %.10f\n", i, error); + totaltime = fann_mstime()-before; + printf("Epochs %8d. Current error: %.10f :: %ld ms by epoch\n", i, error, i ? totaltime/i : 0); } - printf("ticks %d", GetTickCount()-before); + printf("Time spent %ld ms\n", totaltime); fann_destroy(ann); fann_destroy_train(data); From b36858d5e2047a8660efa6bdfbc43528bb11556a Mon Sep 17 00:00:00 2001 From: mingodad Date: Thu, 12 Jul 2018 11:11:14 +0200 Subject: [PATCH 4/7] Add a simple shell script to run example programs without fann been installed --- examples/run-it | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 examples/run-it diff --git a/examples/run-it b/examples/run-it new file mode 100755 index 00000000..85144539 --- /dev/null +++ b/examples/run-it @@ -0,0 +1,2 @@ +export LD_LIBRARY_PATH=$PWD/../build/src:$LD_LIBRARY_PATH +$* From 3cdd94818ea129ee18e9ec69b5df7f4d4b088bc3 Mon Sep 17 00:00:00 2001 From: mingodad Date: Thu, 12 Jul 2018 11:11:39 +0200 Subject: [PATCH 5/7] Change the return value of fann_mstime --- src/fann.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fann.c b/src/fann.c index 92f1d9ed..f341e9b9 100644 --- a/src/fann.c +++ b/src/fann.c @@ -1828,12 +1828,12 @@ void fann_seed_rand() #endif } -long long fann_mstime(void) { +long fann_mstime(void) { struct timeval tv; - long long ust; + long ust; gettimeofday(&tv, NULL); - ust = ((long long)tv.tv_sec)*1000000; + ust = ((long)tv.tv_sec)*1000000; ust += tv.tv_usec; return ust/1000; } From 7e9a91c3f0da3db8ed198aeb0b3464cbdd2eddec Mon Sep 17 00:00:00 2001 From: mingodad Date: Thu, 12 Jul 2018 11:12:32 +0200 Subject: [PATCH 6/7] More refactoring and removing code duplication --- src/fann_train.c | 198 ++++++++----------- src/fann_train_data.c | 427 +++++++++++++++++------------------------ src/parallel_fann.c | 434 +++++++++++++++++++++--------------------- 3 files changed, 469 insertions(+), 590 deletions(-) diff --git a/src/fann_train.c b/src/fann_train.c index 049e6de9..88a5b54c 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -413,34 +413,18 @@ void fann_update_weights(struct fann *ann) { prev_neurons = (layer_it - 1)->first_neuron; } - for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) - { - tmp_error = error_begin[neuron_it - first_neuron] * learning_rate; - num_connections = neuron_it->last_con - neuron_it->first_con; - weights = ann->weights + neuron_it->first_con; - weights_deltas = deltas_begin + neuron_it->first_con; - for(i = 0; i != num_connections; i++) - { - delta_w = tmp_error * prev_neurons[i].value + learning_momentum * weights_deltas[i]; - weights[i] += delta_w ; - weights_deltas[i] = delta_w; - } - } } - else + for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) { - for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) + tmp_error = error_begin[neuron_it - first_neuron] * learning_rate; + num_connections = neuron_it->last_con - neuron_it->first_con; + weights = ann->weights + neuron_it->first_con; + weights_deltas = deltas_begin + neuron_it->first_con; + for(i = 0; i != num_connections; i++) { - tmp_error = error_begin[neuron_it - first_neuron] * learning_rate; - num_connections = neuron_it->last_con - neuron_it->first_con; - weights = ann->weights + neuron_it->first_con; - weights_deltas = deltas_begin + neuron_it->first_con; - for(i = 0; i != num_connections; i++) - { - delta_w = tmp_error * prev_neurons[i].value + learning_momentum * weights_deltas[i]; - weights[i] += delta_w; - weights_deltas[i] = delta_w; - } + delta_w = tmp_error * prev_neurons[i].value + learning_momentum * weights_deltas[i]; + weights[i] += delta_w; + weights_deltas[i] = delta_w; } } } @@ -452,29 +436,36 @@ void fann_update_weights(struct fann *ann) will update all slopes. */ +static int check_alloc_train_slopes(struct fann *ann) +{ + /* if no room allocated for the slope variabels, allocate it now */ + if(ann->train_slopes == NULL) + { + ann->train_slopes = + (fann_type *) calloc(ann->total_connections_allocated, sizeof(fann_type)); + if(ann->train_slopes == NULL) + { + fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); + return 0; + } + } + return 1; +} + void fann_update_slopes_batch(struct fann *ann, struct fann_layer *layer_begin, struct fann_layer *layer_end) { struct fann_neuron *neuron_it, *last_neuron, *prev_neurons, **connections; fann_type tmp_error; unsigned int i, num_connections; + int connection_rate_bte_1 = ann->connection_rate >= 1; /* store some variabels local for fast access */ struct fann_neuron *first_neuron = ann->first_layer->first_neuron; fann_type *error_begin = ann->train_errors; fann_type *slope_begin, *neuron_slope; - /* if no room allocated for the slope variabels, allocate it now */ - if(ann->train_slopes == NULL) - { - ann->train_slopes = - (fann_type *) calloc(ann->total_connections_allocated, sizeof(fann_type)); - if(ann->train_slopes == NULL) - { - fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); - return; - } - } + if(!check_alloc_train_slopes(ann)) return; if(layer_begin == NULL) { @@ -500,36 +491,23 @@ void fann_update_slopes_batch(struct fann *ann, struct fann_layer *layer_begin, printf("layer[%d]\n", layer_begin - ann->first_layer); #endif last_neuron = layer_begin->last_neuron; - if(ann->connection_rate >= 1) + if(connection_rate_bte_1) { if(ann->network_type == FANN_NETTYPE_LAYER) { prev_neurons = (layer_begin - 1)->first_neuron; } - - for(neuron_it = layer_begin->first_neuron; neuron_it != last_neuron; neuron_it++) - { - tmp_error = error_begin[neuron_it - first_neuron]; - neuron_slope = slope_begin + neuron_it->first_con; - num_connections = neuron_it->last_con - neuron_it->first_con; - for(i = 0; i != num_connections; i++) - { - neuron_slope[i] += tmp_error * prev_neurons[i].value; - } - } } - else + for(neuron_it = layer_begin->first_neuron; neuron_it != last_neuron; neuron_it++) { - for(neuron_it = layer_begin->first_neuron; neuron_it != last_neuron; neuron_it++) + tmp_error = error_begin[neuron_it - first_neuron]; + neuron_slope = slope_begin + neuron_it->first_con; + num_connections = neuron_it->last_con - neuron_it->first_con; + if(!connection_rate_bte_1) connections = ann->connections + neuron_it->first_con; + for(i = 0; i != num_connections; i++) { - tmp_error = error_begin[neuron_it - first_neuron]; - neuron_slope = slope_begin + neuron_it->first_con; - num_connections = neuron_it->last_con - neuron_it->first_con; - connections = ann->connections + neuron_it->first_con; - for(i = 0; i != num_connections; i++) - { - neuron_slope[i] += tmp_error * connections[i]->value; - } + neuron_slope[i] += tmp_error * (connection_rate_bte_1 ? + prev_neurons[i].value : connections[i]->value); } } } @@ -544,18 +522,7 @@ void fann_clear_train_arrays(struct fann *ann) unsigned int i; fann_type delta_zero; - /* if no room allocated for the slope variabels, allocate it now - * (calloc clears mem) */ - if(ann->train_slopes == NULL) - { - ann->train_slopes = - (fann_type *) calloc(ann->total_connections_allocated, sizeof(fann_type)); - if(ann->train_slopes == NULL) - { - fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); - return; - } - } + if(!check_alloc_train_slopes(ann)) return; else { memset(ann->train_slopes, 0, (ann->total_connections_allocated) * sizeof(fann_type)); @@ -622,13 +589,17 @@ void fann_update_weights_batch(struct fann *ann, unsigned int num_data, unsigned /* INTERNAL FUNCTION The quickprop training algorithm */ + +#define MK_LOCAL_VARS_COPY(ann)\ + fann_type *train_slopes = ann->train_slopes;\ + fann_type *weights = ann->weights;\ + fann_type *prev_steps = ann->prev_steps;\ + fann_type *prev_train_slopes = ann->prev_train_slopes;\ + void fann_update_weights_quickprop(struct fann *ann, unsigned int num_data, unsigned int first_weight, unsigned int past_end) { - fann_type *train_slopes = ann->train_slopes; - fann_type *weights = ann->weights; - fann_type *prev_steps = ann->prev_steps; - fann_type *prev_train_slopes = ann->prev_train_slopes; + MK_LOCAL_VARS_COPY(ann); fann_type w, prev_step, slope, prev_slope, next_step; @@ -712,10 +683,7 @@ void fann_update_weights_quickprop(struct fann *ann, unsigned int num_data, */ void fann_update_weights_irpropm(struct fann *ann, unsigned int first_weight, unsigned int past_end) { - fann_type *train_slopes = ann->train_slopes; - fann_type *weights = ann->weights; - fann_type *prev_steps = ann->prev_steps; - fann_type *prev_train_slopes = ann->prev_train_slopes; + MK_LOCAL_VARS_COPY(ann); fann_type prev_step, slope, prev_slope, next_step, same_sign; @@ -771,10 +739,7 @@ void fann_update_weights_irpropm(struct fann *ann, unsigned int first_weight, un */ void fann_update_weights_sarprop(struct fann *ann, unsigned int epoch, unsigned int first_weight, unsigned int past_end) { - fann_type *train_slopes = ann->train_slopes; - fann_type *weights = ann->weights; - fann_type *prev_steps = ann->prev_steps; - fann_type *prev_train_slopes = ann->prev_train_slopes; + MK_LOCAL_VARS_COPY(ann); fann_type prev_step, slope, prev_slope, next_step = 0, same_sign; @@ -850,8 +815,10 @@ void fann_update_weights_sarprop(struct fann *ann, unsigned int epoch, unsigned FANN_GET_SET(enum fann_train_enum, training_algorithm) FANN_GET_SET(float, learning_rate) -FANN_EXTERNAL void FANN_API fann_set_activation_function_hidden(struct fann *ann, - enum fann_activationfunc_enum activation_function) +static void fann_set_activation_function_on_hidden(struct fann *ann, + enum fann_activationfunc_enum activation_function, + fann_type steepness, + int isSteep) { struct fann_neuron *last_neuron, *neuron_it; struct fann_layer *layer_it; @@ -862,11 +829,18 @@ FANN_EXTERNAL void FANN_API fann_set_activation_function_hidden(struct fann *ann last_neuron = layer_it->last_neuron; for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) { - neuron_it->activation_function = activation_function; + if(isSteep) neuron_it->activation_steepness = steepness; + else neuron_it->activation_function = activation_function; } } } +FANN_EXTERNAL void FANN_API fann_set_activation_function_hidden(struct fann *ann, + enum fann_activationfunc_enum activation_function) +{ + fann_set_activation_function_on_hidden(ann, activation_function, 0, 0); +} + FANN_EXTERNAL struct fann_layer* FANN_API fann_get_layer(struct fann *ann, int layer) { if(layer <= 0 || layer >= (ann->last_layer - ann->first_layer)) @@ -924,13 +898,13 @@ FANN_EXTERNAL void FANN_API fann_set_activation_function(struct fann *ann, neuron_it->activation_function = activation_function; } -FANN_EXTERNAL void FANN_API fann_set_activation_function_layer(struct fann *ann, - enum fann_activationfunc_enum - activation_function, - int layer) +static void fann_set_activation_function_on_layer(struct fann *ann, + enum fann_activationfunc_enum activation_function, + struct fann_layer * layer_it, fann_type steepness, + int isSteep + ) { struct fann_neuron *last_neuron, *neuron_it; - struct fann_layer *layer_it = fann_get_layer(ann, layer); if(layer_it == NULL) return; @@ -938,39 +912,29 @@ FANN_EXTERNAL void FANN_API fann_set_activation_function_layer(struct fann *ann, last_neuron = layer_it->last_neuron; for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) { - neuron_it->activation_function = activation_function; + if(isSteep) neuron_it->activation_steepness = steepness; + else neuron_it->activation_function = activation_function; } } +FANN_EXTERNAL void FANN_API fann_set_activation_function_layer(struct fann *ann, + enum fann_activationfunc_enum + activation_function, + int layer) +{ + fann_set_activation_function_on_layer(ann, activation_function, fann_get_layer(ann, layer), 0, 0); +} FANN_EXTERNAL void FANN_API fann_set_activation_function_output(struct fann *ann, enum fann_activationfunc_enum activation_function) { - struct fann_neuron *last_neuron, *neuron_it; - struct fann_layer *last_layer = ann->last_layer - 1; - - last_neuron = last_layer->last_neuron; - for(neuron_it = last_layer->first_neuron; neuron_it != last_neuron; neuron_it++) - { - neuron_it->activation_function = activation_function; - } + fann_set_activation_function_on_layer(ann, activation_function, ann->last_layer - 1, 0, 0); } FANN_EXTERNAL void FANN_API fann_set_activation_steepness_hidden(struct fann *ann, fann_type steepness) { - struct fann_neuron *last_neuron, *neuron_it; - struct fann_layer *layer_it; - struct fann_layer *last_layer = ann->last_layer - 1; /* -1 to not update the output layer */ - - for(layer_it = ann->first_layer + 1; layer_it != last_layer; layer_it++) - { - last_neuron = layer_it->last_neuron; - for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) - { - neuron_it->activation_steepness = steepness; - } - } + fann_set_activation_function_on_hidden(ann, 0, steepness, 1); } FANN_EXTERNAL fann_type FANN_API @@ -1003,17 +967,7 @@ FANN_EXTERNAL void FANN_API fann_set_activation_steepness_layer(struct fann *ann fann_type steepness, int layer) { - struct fann_neuron *last_neuron, *neuron_it; - struct fann_layer *layer_it = fann_get_layer(ann, layer); - - if(layer_it == NULL) - return; - - last_neuron = layer_it->last_neuron; - for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) - { - neuron_it->activation_steepness = steepness; - } + fann_set_activation_function_on_layer(ann, 0, fann_get_layer(ann, layer), steepness, 1); } FANN_EXTERNAL void FANN_API fann_set_activation_steepness_output(struct fann *ann, diff --git a/src/fann_train_data.c b/src/fann_train_data.c index d9f4e98b..68a31d8c 100644 --- a/src/fann_train_data.c +++ b/src/fann_train_data.c @@ -102,7 +102,7 @@ FANN_EXTERNAL float FANN_API fann_test_data(struct fann *ann, struct fann_train_ /* * Internal train function */ -float fann_train_epoch_quickprop(struct fann *ann, struct fann_train_data *data) +static float fann_train_epoch_multi(struct fann *ann, struct fann_train_data *data, int update_type) { unsigned int i; @@ -120,36 +120,40 @@ float fann_train_epoch_quickprop(struct fann *ann, struct fann_train_data *data) fann_backpropagate_MSE(ann); fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); } - fann_update_weights_quickprop(ann, data->num_data, 0, ann->total_connections); + switch(update_type) + { + case FANN_TRAIN_RPROP: + fann_update_weights_irpropm(ann, 0, ann->total_connections); + break; + + case FANN_TRAIN_QUICKPROP: + fann_update_weights_quickprop(ann, data->num_data, 0, ann->total_connections); + break; + + case FANN_TRAIN_SARPROP: + fann_update_weights_sarprop(ann, ann->sarprop_epoch, 0, ann->total_connections); + ++(ann->sarprop_epoch); + break; + + case FANN_TRAIN_BATCH: + fann_update_weights_batch(ann, data->num_data, 0, ann->total_connections); + break; + } return fann_get_MSE(ann); } +float fann_train_epoch_quickprop(struct fann *ann, struct fann_train_data *data) +{ + return fann_train_epoch_multi(ann, data, FANN_TRAIN_QUICKPROP); +} + /* * Internal train function */ float fann_train_epoch_irpropm(struct fann *ann, struct fann_train_data *data) { - unsigned int i; - - if(ann->prev_train_slopes == NULL) - { - fann_clear_train_arrays(ann); - } - - fann_reset_MSE(ann); - - for(i = 0; i < data->num_data; i++) - { - fann_run(ann, data->input[i]); - fann_compute_MSE(ann, data->output[i]); - fann_backpropagate_MSE(ann); - fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); - } - - fann_update_weights_irpropm(ann, 0, ann->total_connections); - - return fann_get_MSE(ann); + return fann_train_epoch_multi(ann, data, FANN_TRAIN_RPROP); } /* @@ -157,28 +161,7 @@ float fann_train_epoch_irpropm(struct fann *ann, struct fann_train_data *data) */ float fann_train_epoch_sarprop(struct fann *ann, struct fann_train_data *data) { - unsigned int i; - - if(ann->prev_train_slopes == NULL) - { - fann_clear_train_arrays(ann); - } - - fann_reset_MSE(ann); - - for(i = 0; i < data->num_data; i++) - { - fann_run(ann, data->input[i]); - fann_compute_MSE(ann, data->output[i]); - fann_backpropagate_MSE(ann); - fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); - } - - fann_update_weights_sarprop(ann, ann->sarprop_epoch, 0, ann->total_connections); - - ++(ann->sarprop_epoch); - - return fann_get_MSE(ann); + return fann_train_epoch_multi(ann, data, FANN_TRAIN_SARPROP); } /* @@ -186,21 +169,7 @@ float fann_train_epoch_sarprop(struct fann *ann, struct fann_train_data *data) */ float fann_train_epoch_batch(struct fann *ann, struct fann_train_data *data) { - unsigned int i; - - fann_reset_MSE(ann); - - for(i = 0; i < data->num_data; i++) - { - fann_run(ann, data->input[i]); - fann_compute_MSE(ann, data->output[i]); - fann_backpropagate_MSE(ann); - fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); - } - - fann_update_weights_batch(ann, data->num_data, 0, ann->total_connections); - - return fann_get_MSE(ann); + return fann_train_epoch_multi(ann, data, FANN_TRAIN_BATCH); } /* @@ -252,6 +221,7 @@ FANN_EXTERNAL void FANN_API fann_train_on_data(struct fann *ann, struct fann_tra float error; unsigned int i; int desired_error_reached; + long totaltime = 0; #ifdef DEBUG printf("Training with %s\n", FANN_TRAIN_NAMES[ann->training_algorithm]); @@ -262,6 +232,7 @@ FANN_EXTERNAL void FANN_API fann_train_on_data(struct fann *ann, struct fann_tra printf("Max epochs %8d. Desired error: %.10f.\n", max_epochs, desired_error); } + long start = fann_mstime(); for(i = 1; i <= max_epochs; i++) { /* @@ -279,8 +250,10 @@ FANN_EXTERNAL void FANN_API fann_train_on_data(struct fann *ann, struct fann_tra { if(ann->callback == NULL) { - printf("Epochs %8d. Current error: %.10f. Bit fail %d.\n", i, error, - ann->num_bit_fail); + long elapsed = fann_mstime() - start; + totaltime += elapsed; + printf("Epochs %8d. Current error: %.10f. Bit fail %d and %ld ms per epoch\n", i, error, + ann->num_bit_fail, i ? totaltime/i : 0); } else if(((*ann->callback)(ann, data, max_epochs, epochs_between_reports, desired_error, i)) == -1) @@ -559,8 +532,8 @@ FANN_EXTERNAL struct fann_train_data *FANN_API fann_merge_train_data(struct fann /* * return a copy of a fann_train_data struct */ -FANN_EXTERNAL struct fann_train_data *FANN_API fann_duplicate_train_data(struct fann_train_data - *data) +static struct fann_train_data *fann_duplicate_train_data_len( + struct fann_train_data *data, unsigned int pos, unsigned int length) { unsigned int i; fann_type *data_input, *data_output; @@ -576,7 +549,7 @@ FANN_EXTERNAL struct fann_train_data *FANN_API fann_duplicate_train_data(struct fann_init_error_data((struct fann_error *) dest); dest->error_log = data->error_log; - dest->num_data = data->num_data; + dest->num_data = length; dest->num_input = data->num_input; dest->num_output = data->num_output; dest->input = (fann_type **) calloc(dest->num_data, sizeof(fann_type *)); @@ -602,7 +575,7 @@ FANN_EXTERNAL struct fann_train_data *FANN_API fann_duplicate_train_data(struct fann_destroy_train(dest); return NULL; } - memcpy(data_input, data->input[0], dest->num_input * dest->num_data * sizeof(fann_type)); + memcpy(data_input, data->input[pos], dest->num_input * dest->num_data * sizeof(fann_type)); data_output = (fann_type *) calloc(dest->num_output * dest->num_data, sizeof(fann_type)); if(data_output == NULL) @@ -611,7 +584,7 @@ FANN_EXTERNAL struct fann_train_data *FANN_API fann_duplicate_train_data(struct fann_destroy_train(dest); return NULL; } - memcpy(data_output, data->output[0], dest->num_output * dest->num_data * sizeof(fann_type)); + memcpy(data_output, data->output[pos], dest->num_output * dest->num_data * sizeof(fann_type)); for(i = 0; i != dest->num_data; i++) { @@ -623,75 +596,22 @@ FANN_EXTERNAL struct fann_train_data *FANN_API fann_duplicate_train_data(struct return dest; } +FANN_EXTERNAL struct fann_train_data *FANN_API fann_duplicate_train_data(struct fann_train_data + *data) +{ + return fann_duplicate_train_data_len(data, 0, data->num_data); +} + FANN_EXTERNAL struct fann_train_data *FANN_API fann_subset_train_data(struct fann_train_data *data, unsigned int pos, unsigned int length) { - unsigned int i; - fann_type *data_input, *data_output; - struct fann_train_data *dest = - (struct fann_train_data *) malloc(sizeof(struct fann_train_data)); - - if(dest == NULL) - { - fann_error((struct fann_error*)data, FANN_E_CANT_ALLOCATE_MEM); - return NULL; - } - if(pos > data->num_data || pos+length > data->num_data) { fann_error((struct fann_error*)data, FANN_E_TRAIN_DATA_SUBSET, pos, length, data->num_data); return NULL; } - - fann_init_error_data((struct fann_error *) dest); - dest->error_log = data->error_log; - - dest->num_data = length; - dest->num_input = data->num_input; - dest->num_output = data->num_output; - dest->input = (fann_type **) calloc(dest->num_data, sizeof(fann_type *)); - if(dest->input == NULL) - { - fann_error((struct fann_error*)data, FANN_E_CANT_ALLOCATE_MEM); - fann_destroy_train(dest); - return NULL; - } - - dest->output = (fann_type **) calloc(dest->num_data, sizeof(fann_type *)); - if(dest->output == NULL) - { - fann_error((struct fann_error*)data, FANN_E_CANT_ALLOCATE_MEM); - fann_destroy_train(dest); - return NULL; - } - - data_input = (fann_type *) calloc(dest->num_input * dest->num_data, sizeof(fann_type)); - if(data_input == NULL) - { - fann_error((struct fann_error*)data, FANN_E_CANT_ALLOCATE_MEM); - fann_destroy_train(dest); - return NULL; - } - memcpy(data_input, data->input[pos], dest->num_input * dest->num_data * sizeof(fann_type)); - - data_output = (fann_type *) calloc(dest->num_output * dest->num_data, sizeof(fann_type)); - if(data_output == NULL) - { - fann_error((struct fann_error*)data, FANN_E_CANT_ALLOCATE_MEM); - fann_destroy_train(dest); - return NULL; - } - memcpy(data_output, data->output[pos], dest->num_output * dest->num_data * sizeof(fann_type)); - - for(i = 0; i != dest->num_data; i++) - { - dest->input[i] = data_input; - data_input += dest->num_input; - dest->output[i] = data_output; - data_output += dest->num_output; - } - return dest; + return fann_duplicate_train_data_len(data, pos, length); } FANN_EXTERNAL unsigned int FANN_API fann_length_train_data(struct fann_train_data *data) @@ -952,12 +872,11 @@ struct fann_train_data *fann_read_train_from_fd(FILE * file, const char *filenam { unsigned int num_input, num_output, num_data, i, j; unsigned int line = 1; - struct fann_train_data *data; + struct fann_train_data *data=NULL; if(fscanf(file, "%u %u %u\n", &num_data, &num_input, &num_output) != 3) { - fann_error(NULL, FANN_E_CANT_READ_TD, filename, line); - return NULL; + goto bad_scanf; } line++; @@ -973,9 +892,7 @@ struct fann_train_data *fann_read_train_from_fd(FILE * file, const char *filenam { if(fscanf(file, FANNSCANF " ", &data->input[i][j]) != 1) { - fann_error(NULL, FANN_E_CANT_READ_TD, filename, line); - fann_destroy_train(data); - return NULL; + goto bad_scanf; } } line++; @@ -984,14 +901,16 @@ struct fann_train_data *fann_read_train_from_fd(FILE * file, const char *filenam { if(fscanf(file, FANNSCANF " ", &data->output[i][j]) != 1) { - fann_error(NULL, FANN_E_CANT_READ_TD, filename, line); - fann_destroy_train(data); - return NULL; + goto bad_scanf; } } line++; } return data; +bad_scanf: + fann_error(NULL, FANN_E_CANT_READ_TD, filename, line); + if(data) fann_destroy_train(data); + return NULL; } /* @@ -1014,28 +933,65 @@ int fann_desired_error_reached(struct fann *ann, float desired_error) } #ifndef FIXEDFANN -/* - * Scale data in input vector before feed it to ann based on previously calculated parameters. - */ -FANN_EXTERNAL void FANN_API fann_scale_input( struct fann *ann, fann_type *input_vector ) +static void fann_scale_input_or_output( struct fann *ann, fann_type *io_vector , int isInput, int isDescale) { - unsigned cur_neuron; - if(ann->scale_mean_in == NULL) + unsigned cur_neuron, last_neuron; + float *scale_deviation, *scale_mean, *scale_factor, *scale_new_min; + + if(isInput) + { + last_neuron = ann->num_input; + scale_deviation = ann->scale_deviation_in; + scale_mean = ann->scale_mean_in; + scale_factor = ann->scale_factor_in; + scale_new_min = ann->scale_new_min_in; + } + else + { + last_neuron = ann->num_output; + scale_deviation = ann->scale_deviation_out; + scale_mean = ann->scale_mean_out; + scale_factor = ann->scale_factor_out; + scale_new_min = ann->scale_new_min_out; + } + + if(scale_mean == NULL) { fann_error( (struct fann_error *) ann, FANN_E_SCALE_NOT_PRESENT ); return; } - for( cur_neuron = 0; cur_neuron < ann->num_input; cur_neuron++ ) - if(ann->scale_deviation_in[ cur_neuron ] != 0.0) - input_vector[ cur_neuron ] = - ( - ( input_vector[ cur_neuron ] - ann->scale_mean_in[ cur_neuron ] ) - / ann->scale_deviation_in[ cur_neuron ] - - ( (fann_type)-1.0 ) /* This is old_min */ - ) - * ann->scale_factor_in[ cur_neuron ] - + ann->scale_new_min_in[ cur_neuron ]; + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + if(scale_deviation[ cur_neuron ] != 0.0) + io_vector[ cur_neuron ] = + isDescale ? ( + ( + ( + io_vector[ cur_neuron ] + - scale_new_min[ cur_neuron ] + ) + / scale_factor[ cur_neuron ] + + ( (fann_type)-1.0 ) /* This is old_min */ + ) + * scale_deviation[ cur_neuron ] + + scale_mean[ cur_neuron ] + ) : ( + ( + ( io_vector[ cur_neuron ] - scale_mean[ cur_neuron ] ) + / scale_deviation[ cur_neuron ] + - ( (fann_type)-1.0 ) /* This is old_min */ + ) + * scale_factor[ cur_neuron ] + + scale_new_min[ cur_neuron ] + ); +} + +/* + * Scale data in input vector before feed it to ann based on previously calculated parameters. + */ +FANN_EXTERNAL void FANN_API fann_scale_input( struct fann *ann, fann_type *input_vector ) +{ + fann_scale_input_or_output(ann, input_vector, 1, 0); } /* @@ -1043,23 +999,7 @@ FANN_EXTERNAL void FANN_API fann_scale_input( struct fann *ann, fann_type *input */ FANN_EXTERNAL void FANN_API fann_scale_output( struct fann *ann, fann_type *output_vector ) { - unsigned cur_neuron; - if(ann->scale_mean_in == NULL) - { - fann_error( (struct fann_error *) ann, FANN_E_SCALE_NOT_PRESENT ); - return; - } - - for( cur_neuron = 0; cur_neuron < ann->num_output; cur_neuron++ ) - if(ann->scale_deviation_out[ cur_neuron ] != 0.0) - output_vector[ cur_neuron ] = - ( - ( output_vector[ cur_neuron ] - ann->scale_mean_out[ cur_neuron ] ) - / ann->scale_deviation_out[ cur_neuron ] - - ( (fann_type)-1.0 ) /* This is old_min */ - ) - * ann->scale_factor_out[ cur_neuron ] - + ann->scale_new_min_out[ cur_neuron ]; + fann_scale_input_or_output(ann, output_vector, 0, 0); } /* @@ -1067,26 +1007,7 @@ FANN_EXTERNAL void FANN_API fann_scale_output( struct fann *ann, fann_type *outp */ FANN_EXTERNAL void FANN_API fann_descale_input( struct fann *ann, fann_type *input_vector ) { - unsigned cur_neuron; - if(ann->scale_mean_in == NULL) - { - fann_error( (struct fann_error *) ann, FANN_E_SCALE_NOT_PRESENT ); - return; - } - - for( cur_neuron = 0; cur_neuron < ann->num_input; cur_neuron++ ) - if(ann->scale_deviation_in[ cur_neuron ] != 0.0) - input_vector[ cur_neuron ] = - ( - ( - input_vector[ cur_neuron ] - - ann->scale_new_min_in[ cur_neuron ] - ) - / ann->scale_factor_in[ cur_neuron ] - + ( (fann_type)-1.0 ) /* This is old_min */ - ) - * ann->scale_deviation_in[ cur_neuron ] - + ann->scale_mean_in[ cur_neuron ]; + fann_scale_input_or_output(ann, input_vector, 1, 1); } /* @@ -1094,26 +1015,7 @@ FANN_EXTERNAL void FANN_API fann_descale_input( struct fann *ann, fann_type *inp */ FANN_EXTERNAL void FANN_API fann_descale_output( struct fann *ann, fann_type *output_vector ) { - unsigned cur_neuron; - if(ann->scale_mean_in == NULL) - { - fann_error( (struct fann_error *) ann, FANN_E_SCALE_NOT_PRESENT ); - return; - } - - for( cur_neuron = 0; cur_neuron < ann->num_output; cur_neuron++ ) - if(ann->scale_deviation_out[ cur_neuron ] != 0.0) - output_vector[ cur_neuron ] = - ( - ( - output_vector[ cur_neuron ] - - ann->scale_new_min_out[ cur_neuron ] - ) - / ann->scale_factor_out[ cur_neuron ] - + ( (fann_type)-1.0 ) /* This is old_min */ - ) - * ann->scale_deviation_out[ cur_neuron ] - + ann->scale_mean_out[ cur_neuron ]; + fann_scale_input_or_output(ann, output_vector, 0, 1); } /* @@ -1160,49 +1062,72 @@ FANN_EXTERNAL void FANN_API fann_descale_train( struct fann *ann, struct fann_tr } } -#define SCALE_RESET( what, where, default_value ) \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->what##_##where[ cur_neuron ] = ( default_value ); - -#define SCALE_SET_PARAM( where ) \ - /* Calculate mean: sum(x)/length */ \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->scale_mean_##where[ cur_neuron ] = 0.0f; \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - for( cur_sample = 0; cur_sample < data->num_data; cur_sample++ ) \ - ann->scale_mean_##where[ cur_neuron ] += (float)data->where##put[ cur_sample ][ cur_neuron ];\ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->scale_mean_##where[ cur_neuron ] /= (float)data->num_data; \ - /* Calculate deviation: sqrt(sum((x-mean)^2)/length) */ \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->scale_deviation_##where[ cur_neuron ] = 0.0f; \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - for( cur_sample = 0; cur_sample < data->num_data; cur_sample++ ) \ - ann->scale_deviation_##where[ cur_neuron ] += \ - /* Another local variable in macro? Oh no! */ \ - ( \ - (float)data->where##put[ cur_sample ][ cur_neuron ] \ - - ann->scale_mean_##where[ cur_neuron ] \ - ) \ - * \ - ( \ - (float)data->where##put[ cur_sample ][ cur_neuron ] \ - - ann->scale_mean_##where[ cur_neuron ] \ - ); \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->scale_deviation_##where[ cur_neuron ] = \ - sqrtf( ann->scale_deviation_##where[ cur_neuron ] / (float)data->num_data ); \ - /* Calculate factor: (new_max-new_min)/(old_max(1)-old_min(-1)) */ \ - /* Looks like we dont need whole array of factors? */ \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->scale_factor_##where[ cur_neuron ] = \ - ( new_##where##put_max - new_##where##put_min ) \ - / \ - ( 1.0f - ( -1.0f ) ); \ - /* Copy new minimum. */ \ - /* Looks like we dont need whole array of new minimums? */ \ - for( cur_neuron = 0; cur_neuron < ann->num_##where##put; cur_neuron++ ) \ - ann->scale_new_min_##where[ cur_neuron ] = new_##where##put_min; +static void scale_reset_on(unsigned last_neuron, float *values, float default_value) +{ + unsigned cur_neuron; + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + values[ cur_neuron ] = ( default_value ); +} + +#define SCALE_RESET( what, where, default_value ) \ + scale_reset_on(ann->num_##where##put, ann->what##_##where, default_value); + +static void scale_set_param( unsigned last_neuron, float *scale_mean, + float *scale_deviation, float *scale_factor, + float *scale_new_min, float new_min, + float new_max, + fann_type **data_io, unsigned int num_data + ) +{ + unsigned cur_neuron, cur_sample; + /* Calculate mean: sum(x)/length */ + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + scale_mean[ cur_neuron ] = 0.0f; + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + for( cur_sample = 0; cur_sample < num_data; cur_sample++ ) + scale_mean[ cur_neuron ] += (float)data_io[ cur_sample ][ cur_neuron ]; + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + scale_mean[ cur_neuron ] /= (float)num_data; + /* Calculate deviation: sqrt(sum((x-mean)^2)/length) */ + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + scale_deviation[ cur_neuron ] = 0.0f; + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + for( cur_sample = 0; cur_sample < num_data; cur_sample++ ) + scale_deviation[ cur_neuron ] += + /* Another local variable in macro? Oh no! */ + ( + (float)data_io[ cur_sample ][ cur_neuron ] + - scale_mean[ cur_neuron ] + ) + * + ( + (float)data_io[ cur_sample ][ cur_neuron ] + - scale_mean[ cur_neuron ] + ); + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + scale_deviation[ cur_neuron ] = + sqrtf( scale_deviation[ cur_neuron ] / (float)num_data ); + /* Calculate factor: (new_max-new_min)/(old_max(1)-old_min(-1)) */ + /* Looks like we dont need whole array of factors? */ + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + scale_factor[ cur_neuron ] = + ( new_max - new_min ) + / + ( 1.0f - ( -1.0f ) ); + /* Copy new minimum. */ + /* Looks like we dont need whole array of new minimums? */ + for( cur_neuron = 0; cur_neuron < last_neuron; cur_neuron++ ) + scale_new_min[ cur_neuron ] = new_min; +} + +#define SCALE_SET_PARAM( where ) \ + scale_set_param( ann->num_##where##put, ann->scale_mean_##where, \ + ann->scale_deviation_##where, ann->scale_factor_##where, \ + ann->scale_new_min_##where, new_##where##put_min, \ + new_##where##put_max, \ + data->where##put, data->num_data \ + ) + FANN_EXTERNAL int FANN_API fann_set_input_scaling_params( struct fann *ann, diff --git a/src/parallel_fann.c b/src/parallel_fann.c index 7278563b..9d93dace 100644 --- a/src/parallel_fann.c +++ b/src/parallel_fann.c @@ -27,9 +27,9 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_batch_parallel(struct fann *ann, s ann_vect[i]=fann_copy(ann); } - //parallel computing of the updates + //parallel computing of the updates - #pragma omp for schedule(static) + #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); @@ -40,7 +40,7 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_batch_parallel(struct fann *ann, s } } - //parallel update of the weights + //parallel update of the weights { const unsigned int num_data=data->num_data; const unsigned int first_weight=0; @@ -52,20 +52,20 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_batch_parallel(struct fann *ann, s #pragma omp parallel { #pragma omp for schedule(static) - for(i=first_weight; i < (int)past_end; i++) + for(i=first_weight; i < (int)past_end; i++) + { + fann_type temp_slopes=0.0; + unsigned int k; + fann_type *train_slopes; + for(k=0;ktrain_slopes; - temp_slopes+= train_slopes[i]; - train_slopes[i]=0.0; - } - weights[i] += temp_slopes*epsilon; + train_slopes=ann_vect[k]->train_slopes; + temp_slopes+= train_slopes[i]; + train_slopes[i]=0.0; } + weights[i] += temp_slopes*epsilon; } + } } //merge of MSEs for(i=0;i<(int)threadnumb;++i) @@ -106,10 +106,10 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_irpropm_parallel(struct fann *ann, ann_vect[i]=fann_copy(ann); } - //parallel computing of the updates + //parallel computing of the updates - #pragma omp for schedule(static) + #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); @@ -121,16 +121,16 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_irpropm_parallel(struct fann *ann, } { - fann_type *weights = ann->weights; - fann_type *prev_steps = ann->prev_steps; - fann_type *prev_train_slopes = ann->prev_train_slopes; + fann_type *weights = ann->weights; + fann_type *prev_steps = ann->prev_steps; + fann_type *prev_train_slopes = ann->prev_train_slopes; - fann_type next_step; + fann_type next_step; - const float increase_factor = ann->rprop_increase_factor; //1.2; - const float decrease_factor = ann->rprop_decrease_factor; //0.5; - const float delta_min = ann->rprop_delta_min; //0.0; - const float delta_max = ann->rprop_delta_max; //50.0; + const float increase_factor = ann->rprop_increase_factor; //1.2; + const float decrease_factor = ann->rprop_decrease_factor; //0.5; + const float delta_min = ann->rprop_delta_min; //0.0; + const float delta_max = ann->rprop_delta_max; //50.0; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; @@ -139,52 +139,52 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_irpropm_parallel(struct fann *ann, #pragma omp parallel private(next_step) { #pragma omp for schedule(static) - for(i=first_weight; i < (int)past_end; i++) + for(i=first_weight; i < (int)past_end; i++) + { + fann_type prev_slope, same_sign; + const fann_type prev_step = fann_max(prev_steps[i], (fann_type) 0.0001); // prev_step may not be zero because then the training will stop + + fann_type temp_slopes=0.0; + unsigned int k; + fann_type *train_slopes; + for(k=0;ktrain_slopes; - temp_slopes+= train_slopes[i]; - train_slopes[i]=0.0; - } - - prev_slope = prev_train_slopes[i]; - - same_sign = prev_slope * temp_slopes; - - if(same_sign >= 0.0) - next_step = fann_min(prev_step * increase_factor, delta_max); - else - { - next_step = fann_max(prev_step * decrease_factor, delta_min); - temp_slopes = 0; - } - - if(temp_slopes < 0) - { - weights[i] -= next_step; - if(weights[i] < -1500) - weights[i] = -1500; - } - else - { - weights[i] += next_step; - if(weights[i] > 1500) - weights[i] = 1500; - } - - // update global data arrays - prev_steps[i] = next_step; - prev_train_slopes[i] = temp_slopes; + train_slopes=ann_vect[k]->train_slopes; + temp_slopes+= train_slopes[i]; + train_slopes[i]=0.0; + } + + prev_slope = prev_train_slopes[i]; + same_sign = prev_slope * temp_slopes; + + if(same_sign >= 0.0) + next_step = fann_min(prev_step * increase_factor, delta_max); + else + { + next_step = fann_max(prev_step * decrease_factor, delta_min); + temp_slopes = 0; + } + + if(temp_slopes < 0) + { + weights[i] -= next_step; + if(weights[i] < -1500) + weights[i] = -1500; + } + else + { + weights[i] += next_step; + if(weights[i] > 1500) + weights[i] = 1500; } + + // update global data arrays + prev_steps[i] = next_step; + prev_train_slopes[i] = temp_slopes; + } + } } //merge of MSEs @@ -226,9 +226,9 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_quickprop_parallel(struct fann *an ann_vect[i]=fann_copy(ann); } - //parallel computing of the updates + //parallel computing of the updates - #pragma omp for schedule(static) + #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); @@ -239,88 +239,88 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_quickprop_parallel(struct fann *an } } - { - fann_type *weights = ann->weights; - fann_type *prev_steps = ann->prev_steps; - fann_type *prev_train_slopes = ann->prev_train_slopes; + { + fann_type *weights = ann->weights; + fann_type *prev_steps = ann->prev_steps; + fann_type *prev_train_slopes = ann->prev_train_slopes; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; - fann_type w=0.0, next_step; + fann_type w=0.0, next_step; - const float epsilon = ann->learning_rate / data->num_data; - const float decay = ann->quickprop_decay; /*-0.0001;*/ - const float mu = ann->quickprop_mu; /*1.75; */ - const float shrink_factor = (float) (mu / (1.0 + mu)); + const float epsilon = ann->learning_rate / data->num_data; + const float decay = ann->quickprop_decay; /*-0.0001;*/ + const float mu = ann->quickprop_mu; /*1.75; */ + const float shrink_factor = (float) (mu / (1.0 + mu)); omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(w, next_step) { #pragma omp for schedule(static) - for(i=first_weight; i < (int)past_end; i++) + for(i=first_weight; i < (int)past_end; i++) + { + fann_type temp_slopes=0.0; + unsigned int k; + fann_type *train_slopes; + fann_type prev_step, prev_slope; + + w = weights[i]; + for(k=0;ktrain_slopes; - temp_slopes+= train_slopes[i]; - train_slopes[i]=0.0; - } - temp_slopes+= decay * w; - - prev_step = prev_steps[i]; - prev_slope = prev_train_slopes[i]; - - next_step = 0.0; - - - /* The step must always be in direction opposite to the slope. */ - if(prev_step > 0.001) - { - /* If last step was positive... */ - if(temp_slopes > 0.0) /* Add in linear term if current slope is still positive. */ - next_step += epsilon * temp_slopes; - - /*If current slope is close to or larger than prev slope... */ - if(temp_slopes > (shrink_factor * prev_slope)) - next_step += mu * prev_step; /* Take maximum size negative step. */ - else - next_step += prev_step * temp_slopes / (prev_slope - temp_slopes); /* Else, use quadratic estimate. */ - } - else if(prev_step < -0.001) - { - /* If last step was negative... */ - if(temp_slopes < 0.0) /* Add in linear term if current slope is still negative. */ - next_step += epsilon * temp_slopes; - - /* If current slope is close to or more neg than prev slope... */ - if(temp_slopes < (shrink_factor * prev_slope)) - next_step += mu * prev_step; /* Take maximum size negative step. */ - else - next_step += prev_step * temp_slopes / (prev_slope - temp_slopes); /* Else, use quadratic estimate. */ - } - else /* Last step was zero, so use only linear term. */ - next_step += epsilon * temp_slopes; + train_slopes=ann_vect[k]->train_slopes; + temp_slopes+= train_slopes[i]; + train_slopes[i]=0.0; + } + temp_slopes+= decay * w; - /* update global data arrays */ - prev_steps[i] = next_step; - prev_train_slopes[i] = temp_slopes; + prev_step = prev_steps[i]; + prev_slope = prev_train_slopes[i]; - w += next_step; + next_step = 0.0; - if(w > 1500) - weights[i] = 1500; - else if(w < -1500) - weights[i] = -1500; + + /* The step must always be in direction opposite to the slope. */ + if(prev_step > 0.001) + { + /* If last step was positive... */ + if(temp_slopes > 0.0) /* Add in linear term if current slope is still positive. */ + next_step += epsilon * temp_slopes; + + /*If current slope is close to or larger than prev slope... */ + if(temp_slopes > (shrink_factor * prev_slope)) + next_step += mu * prev_step; /* Take maximum size negative step. */ + else + next_step += prev_step * temp_slopes / (prev_slope - temp_slopes); /* Else, use quadratic estimate. */ + } + else if(prev_step < -0.001) + { + /* If last step was negative... */ + if(temp_slopes < 0.0) /* Add in linear term if current slope is still negative. */ + next_step += epsilon * temp_slopes; + + /* If current slope is close to or more neg than prev slope... */ + if(temp_slopes < (shrink_factor * prev_slope)) + next_step += mu * prev_step; /* Take maximum size negative step. */ else - weights[i] = w; + next_step += prev_step * temp_slopes / (prev_slope - temp_slopes); /* Else, use quadratic estimate. */ } + else /* Last step was zero, so use only linear term. */ + next_step += epsilon * temp_slopes; + + /* update global data arrays */ + prev_steps[i] = next_step; + prev_train_slopes[i] = temp_slopes; + + w += next_step; + + if(w > 1500) + weights[i] = 1500; + else if(w < -1500) + weights[i] = -1500; + else + weights[i] = w; + } } } //merge of MSEs @@ -362,9 +362,9 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_sarprop_parallel(struct fann *ann, ann_vect[i]=fann_copy(ann); } - //parallel computing of the updates + //parallel computing of the updates - #pragma omp for schedule(static) + #pragma omp for schedule(static) for(i = 0; i < (int)data->num_data; i++) { j=omp_get_thread_num(); @@ -375,106 +375,106 @@ FANN_EXTERNAL float FANN_API fann_train_epoch_sarprop_parallel(struct fann *ann, } } - { - fann_type *weights = ann->weights; - fann_type *prev_steps = ann->prev_steps; - fann_type *prev_train_slopes = ann->prev_train_slopes; + { + fann_type *weights = ann->weights; + fann_type *prev_steps = ann->prev_steps; + fann_type *prev_train_slopes = ann->prev_train_slopes; const unsigned int first_weight=0; const unsigned int past_end=ann->total_connections; const unsigned int epoch=ann->sarprop_epoch; - fann_type next_step; - - /* These should be set from variables */ - const float increase_factor = ann->rprop_increase_factor; /*1.2; */ - const float decrease_factor = ann->rprop_decrease_factor; /*0.5; */ - /* TODO: why is delta_min 0.0 in iRprop? SARPROP uses 1x10^-6 (Braun and Riedmiller, 1993) */ - const float delta_min = 0.000001f; - const float delta_max = ann->rprop_delta_max; /*50.0; */ - const float weight_decay_shift = ann->sarprop_weight_decay_shift; /* ld 0.01 = -6.644 */ - const float step_error_threshold_factor = ann->sarprop_step_error_threshold_factor; /* 0.1 */ - const float step_error_shift = ann->sarprop_step_error_shift; /* ld 3 = 1.585 */ - const float T = ann->sarprop_temperature; + fann_type next_step; + + /* These should be set from variables */ + const float increase_factor = ann->rprop_increase_factor; /*1.2; */ + const float decrease_factor = ann->rprop_decrease_factor; /*0.5; */ + /* TODO: why is delta_min 0.0 in iRprop? SARPROP uses 1x10^-6 (Braun and Riedmiller, 1993) */ + const float delta_min = 0.000001f; + const float delta_max = ann->rprop_delta_max; /*50.0; */ + const float weight_decay_shift = ann->sarprop_weight_decay_shift; /* ld 0.01 = -6.644 */ + const float step_error_threshold_factor = ann->sarprop_step_error_threshold_factor; /* 0.1 */ + const float step_error_shift = ann->sarprop_step_error_shift; /* ld 3 = 1.585 */ + const float T = ann->sarprop_temperature; float MSE, RMSE; - //merge of MSEs - for(i=0;i<(int)threadnumb;++i) - { - ann->MSE_value+= ann_vect[i]->MSE_value; - ann->num_MSE+=ann_vect[i]->num_MSE; - } + //merge of MSEs + for(i=0;i<(int)threadnumb;++i) + { + ann->MSE_value+= ann_vect[i]->MSE_value; + ann->num_MSE+=ann_vect[i]->num_MSE; + } - MSE = fann_get_MSE(ann); - RMSE = sqrtf(MSE); + MSE = fann_get_MSE(ann); + RMSE = sqrtf(MSE); - /* for all weights; TODO: are biases included? */ + /* for all weights; TODO: are biases included? */ omp_set_dynamic(0); omp_set_num_threads(threadnumb); #pragma omp parallel private(next_step) { #pragma omp for schedule(static) - for(i=first_weight; i < (int)past_end; i++) + for(i=first_weight; i < (int)past_end; i++) + { + /* TODO: confirm whether 1x10^-6 == delta_min is really better */ + const fann_type prev_step = fann_max(prev_steps[i], (fann_type) 0.000001); /* prev_step may not be zero because then the training will stop */ + + /* calculate SARPROP slope; TODO: better as new error function? (see SARPROP paper)*/ + fann_type prev_slope, same_sign; + fann_type temp_slopes=0.0; + unsigned int k; + fann_type *train_slopes; + for(k=0;ktrain_slopes; - temp_slopes+= train_slopes[i]; - train_slopes[i]=0.0; - } - temp_slopes= -temp_slopes - weights[i] * (fann_type)fann_exp2(-T * epoch + weight_decay_shift); - - next_step=0.0; - - /* TODO: is prev_train_slopes[i] 0.0 in the beginning? */ - prev_slope = prev_train_slopes[i]; - - same_sign = prev_slope * temp_slopes; - - if(same_sign > 0.0) - { - next_step = fann_min(prev_step * increase_factor, delta_max); - /* TODO: are the signs inverted? see differences between SARPROP paper and iRprop */ - if (temp_slopes < 0.0) - weights[i] += next_step; - else - weights[i] -= next_step; - } - else if(same_sign < 0.0) - { - #ifndef RAND_MAX - #define RAND_MAX 0x7fffffff - #endif - if(prev_step < step_error_threshold_factor * MSE) - next_step = prev_step * decrease_factor + (float)rand() / RAND_MAX * RMSE * (fann_type)fann_exp2(-T * epoch + step_error_shift); - else - next_step = fann_max(prev_step * decrease_factor, delta_min); - - temp_slopes = 0.0; - } - else - { - if(temp_slopes < 0.0) - weights[i] += prev_step; - else - weights[i] -= prev_step; - } + train_slopes=ann_vect[k]->train_slopes; + temp_slopes+= train_slopes[i]; + train_slopes[i]=0.0; + } + temp_slopes= -temp_slopes - weights[i] * (fann_type)fann_exp2(-T * epoch + weight_decay_shift); - /* update global data arrays */ - prev_steps[i] = next_step; - prev_train_slopes[i] = temp_slopes; + next_step=0.0; + /* TODO: is prev_train_slopes[i] 0.0 in the beginning? */ + prev_slope = prev_train_slopes[i]; + + same_sign = prev_slope * temp_slopes; + + if(same_sign > 0.0) + { + next_step = fann_min(prev_step * increase_factor, delta_max); + /* TODO: are the signs inverted? see differences between SARPROP paper and iRprop */ + if (temp_slopes < 0.0) + weights[i] += next_step; + else + weights[i] -= next_step; + } + else if(same_sign < 0.0) + { + #ifndef RAND_MAX + #define RAND_MAX 0x7fffffff + #endif + if(prev_step < step_error_threshold_factor * MSE) + next_step = prev_step * decrease_factor + (float)rand() / RAND_MAX * RMSE * (fann_type)fann_exp2(-T * epoch + step_error_shift); + else + next_step = fann_max(prev_step * decrease_factor, delta_min); + + temp_slopes = 0.0; } + else + { + if(temp_slopes < 0.0) + weights[i] += prev_step; + else + weights[i] -= prev_step; + } + + /* update global data arrays */ + prev_steps[i] = next_step; + prev_train_slopes[i] = temp_slopes; + + } } - } + } ++(ann->sarprop_epoch); From 8983fd6162a73b94679fe51b0e61ac08b85e2208 Mon Sep 17 00:00:00 2001 From: mingodad Date: Thu, 12 Jul 2018 11:14:18 +0200 Subject: [PATCH 7/7] Add a shell script using https://github.com/nachose/Duplo to find duplicated code --- src/myduplo.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 src/myduplo.sh diff --git a/src/myduplo.sh b/src/myduplo.sh new file mode 100755 index 00000000..618501ae --- /dev/null +++ b/src/myduplo.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +rm -rf duplo_list.txt + +#find -name "*.cpp" ! -name "moc_*.cpp" >> duplo_list.txt +find -name "*.c" >> duplo_list.txt +find -name "*.h" >> duplo_list.txt + +duplo duplo_list.txt duplo_result.txt