diff --git a/src/fann.c b/src/fann.c index 9af7e388..32c3f3b1 100644 --- a/src/fann.c +++ b/src/fann.c @@ -769,6 +769,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann * ann, fann_type * input) case FANN_LINEAR_PIECE_SYMMETRIC: neuron_it->value = (fann_type)((neuron_sum < -multiplier) ? -multiplier : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; + case FANN_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : 0); + break; + case FANN_LEAKY_RELU: + neuron_it->value = (fann_type)(neuron_sum > 0 ? neuron_sum : neuron_sum / 100.0); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: @@ -894,6 +900,7 @@ FANN_EXTERNAL struct fann* FANN_API fann_copy(struct fann* orig) copy->learning_rate = orig->learning_rate; copy->learning_momentum = orig->learning_momentum; + copy->learning_l2_norm = orig->learning_l2_norm; copy->connection_rate = orig->connection_rate; copy->network_type = orig->network_type; copy->num_MSE = orig->num_MSE; @@ -1284,6 +1291,7 @@ FANN_EXTERNAL void FANN_API fann_print_parameters(struct fann *ann) printf("Bit fail limit :%8.3f\n", ann->bit_fail_limit); printf("Learning rate :%8.3f\n", ann->learning_rate); printf("Learning momentum :%8.3f\n", ann->learning_momentum); + printf("Learning l2 norm :%8.3f\n", ann->learning_l2_norm); printf("Quickprop decay :%11.6f\n", ann->quickprop_decay); printf("Quickprop mu :%8.3f\n", ann->quickprop_mu); printf("RPROP increase factor :%8.3f\n", ann->rprop_increase_factor); @@ -1442,6 +1450,74 @@ FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct f } } +FANN_EXTERNAL fann_type FANN_API fann_get_l1_norm(struct fann *ann) +{ + struct fann_neuron *first_neuron; + struct fann_layer *layer_it; + struct fann_neuron *neuron_it; + unsigned int idx; + unsigned int source_index; + fann_type l1_term; + + first_neuron = ann->first_layer->first_neuron; + + source_index = 0; + l1_term = 0.f; + + /* The following assumes that the last unused bias has no connections */ + + /* for each layer */ + for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++){ + /* for each neuron */ + for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++){ + /* for each connection */ + for (idx = neuron_it->first_con; idx < neuron_it->last_con; idx++){ + /* Assign the source, destination and weight */ + l1_term += fabs(ann->weights[source_index]); + + source_index++; + } + } + } + + return l1_term; +} + +FANN_EXTERNAL fann_type FANN_API fann_get_l2_norm(struct fann *ann) +{ + struct fann_neuron *first_neuron; + struct fann_layer *layer_it; + struct fann_neuron *neuron_it; + unsigned int idx; + unsigned int source_index; + fann_type l2_term; + + first_neuron = ann->first_layer->first_neuron; + + source_index = 0; + l2_term = 0.f; + + /* The following assumes that the last unused bias has no connections */ + + /* for each layer */ + for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++){ + /* for each neuron */ + for(neuron_it = layer_it->first_neuron; neuron_it != layer_it->last_neuron; neuron_it++){ + /* for each connection */ + for (idx = neuron_it->first_con; idx < neuron_it->last_con; idx++){ + /* Assign the source, destination and weight */ + l2_term += ann->weights[source_index] * ann->weights[source_index]; + + source_index++; + } + } + } + + return sqrt(l2_term); +} + + + FANN_EXTERNAL void FANN_API fann_set_weight_array(struct fann *ann, struct fann_connection *connections, unsigned int num_connections) { @@ -1585,6 +1661,7 @@ struct fann *fann_allocate_structure(unsigned int num_layers) ann->errstr = NULL; ann->learning_rate = 0.7f; ann->learning_momentum = 0.0; + ann->learning_l2_norm = 0.0; ann->total_neurons = 0; ann->total_connections = 0; ann->num_input = 0; diff --git a/src/fann_io.c b/src/fann_io.c index f17667b9..55c0d50c 100644 --- a/src/fann_io.c +++ b/src/fann_io.c @@ -181,6 +181,7 @@ int fann_save_internal_fd(struct fann *ann, FILE * conf, const char *configurati fprintf(conf, "network_type=%u\n", ann->network_type); fprintf(conf, "learning_momentum=%f\n", ann->learning_momentum); + fprintf(conf, "learning_l2_norm=%f\n", ann->learning_l2_norm); fprintf(conf, "training_algorithm=%u\n", ann->training_algorithm); fprintf(conf, "train_error_function=%u\n", ann->train_error_function); fprintf(conf, "train_stop_function=%u\n", ann->train_stop_function); @@ -443,6 +444,7 @@ struct fann *fann_create_from_fd(FILE * conf, const char *configuration_file) fann_scanf("%u", "network_type", &tmpVal); ann->network_type = (enum fann_nettype_enum)tmpVal; fann_scanf("%f", "learning_momentum", &ann->learning_momentum); + fann_scanf("%f", "learning_l2_norm", &ann->learning_l2_norm); fann_scanf("%u", "training_algorithm", &tmpVal); ann->training_algorithm = (enum fann_train_enum)tmpVal; fann_scanf("%u", "train_error_function", &tmpVal); diff --git a/src/fann_train.c b/src/fann_train.c index 049e6de9..fcff1bb0 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -46,6 +46,8 @@ fann_type fann_activation_derived(unsigned int activation_function, case FANN_SIGMOID_STEPWISE: value = fann_clip(value, 0.01f, 0.99f); return (fann_type) fann_sigmoid_derive(steepness, value); + case FANN_SIGMOID_SYMMETRIC_LECUN: + return (fann_type) fann_sigmoid_symmetric_lecun_derive(steepness, value); case FANN_SIGMOID_SYMMETRIC: case FANN_SIGMOID_SYMMETRIC_STEPWISE: value = fann_clip(value, -0.98f, 0.98f); @@ -72,6 +74,10 @@ fann_type fann_activation_derived(unsigned int activation_function, return (fann_type) fann_cos_derive(steepness, sum); case FANN_THRESHOLD: fann_error(NULL, FANN_E_CANT_TRAIN_ACTIVATION); + case FANN_RELU: + return (fann_type) fann_relu_derive(steepness, sum); + case FANN_LEAKY_RELU: + return (fann_type) fann_leaky_relu_derive(steepness, sum); } return 0; } @@ -133,6 +139,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ case FANN_LINEAR_PIECE: case FANN_SIN: case FANN_COS: + case FANN_RELU: + case FANN_LEAKY_RELU: break; } @@ -154,6 +162,55 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron* neuron, fann_typ return neuron_diff; } +fann_type fann_update_MSE_lw(struct fann *ann, struct fann_neuron* neuron, fann_type neuron_diff, fann_type label_weight) +{ + float neuron_diff2; + + switch (neuron->activation_function) + { + case FANN_LINEAR_PIECE_SYMMETRIC: + case FANN_THRESHOLD_SYMMETRIC: + case FANN_SIGMOID_SYMMETRIC: + case FANN_SIGMOID_SYMMETRIC_STEPWISE: + case FANN_ELLIOT_SYMMETRIC: + case FANN_GAUSSIAN_SYMMETRIC: + case FANN_SIN_SYMMETRIC: + case FANN_COS_SYMMETRIC: + neuron_diff /= (fann_type)2.0; + break; + case FANN_THRESHOLD: + case FANN_LINEAR: + case FANN_SIGMOID: + case FANN_SIGMOID_STEPWISE: + case FANN_GAUSSIAN: + case FANN_GAUSSIAN_STEPWISE: + case FANN_ELLIOT: + case FANN_LINEAR_PIECE: + case FANN_SIN: + case FANN_COS: + break; + } + +#ifdef FIXEDFANN + neuron_diff2 = + (neuron_diff / (float) ann->multiplier) * (neuron_diff / (float) ann->multiplier); +#else + neuron_diff2 = (float) (neuron_diff * neuron_diff); +#endif + + ann->MSE_value += neuron_diff2 * label_weight; + + /*printf("neuron_diff %f = (%f - %f)[/2], neuron_diff2=%f, sum=%f, MSE_value=%f, num_MSE=%d\n", neuron_diff, *desired_output, neuron_value, neuron_diff2, last_layer_begin->sum, ann->MSE_value, ann->num_MSE); */ + if(fann_abs(neuron_diff) >= ann->bit_fail_limit) + { + ann->num_bit_fail++; + } + + return neuron_diff; +} + + + /* Tests the network. */ FANN_EXTERNAL fann_type *FANN_API fann_test(struct fann *ann, fann_type * input, @@ -282,6 +339,117 @@ void fann_compute_MSE(struct fann *ann, fann_type * desired_output) } } +void fann_compute_MSE_gradient(struct fann *ann, fann_type * input, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*), void* errorFuncdata) +{ + fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0; + struct fann_neuron *last_layer_begin = (ann->last_layer - 1)->first_neuron; + const struct fann_neuron *last_layer_end = last_layer_begin + ann->num_output; + const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; + + /* if no room allocated for the error variabels, allocate it now */ + if(ann->train_errors == NULL) + { + ann->train_errors = (fann_type *) calloc(ann->total_neurons, sizeof(fann_type)); + if(ann->train_errors == NULL) + { + fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); + return; + } + } + else + { + /* clear the error variabels */ + memset(ann->train_errors, 0, (ann->total_neurons) * sizeof(fann_type)); + } + error_begin = ann->train_errors; + +#ifdef DEBUGTRAIN + printf("\ncalculate errors\n"); +#endif + /* calculate the error and place it in the output layer */ + error_it = error_begin + (last_layer_begin - first_neuron); + + int i=0; + for(; last_layer_begin != last_layer_end; last_layer_begin++) + { + neuron_value = last_layer_begin->value; + //neuron_diff = -(2.f*neuron_value + *desired_output); + neuron_diff = errorFunction(input, ann->output, i, errorFuncdata); + + *error_it = fann_activation_derived(last_layer_begin->activation_function, + last_layer_begin->activation_steepness, neuron_value, + last_layer_begin->sum) * neuron_diff; + //printf("DEBUG _mse_grad %lf %lf %lf %lf\n", *error_it, neuron_diff, last_layer_begin->activation_function, last_layer_begin->activation_steepness ); + //desired_output++; + error_it++; + + ann->num_MSE++; + i++; + } +} + + + +void fann_compute_MSE_lw(struct fann *ann, fann_type * desired_output, fann_type label_weight) +{ + fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0; + struct fann_neuron *last_layer_begin = (ann->last_layer - 1)->first_neuron; + const struct fann_neuron *last_layer_end = last_layer_begin + ann->num_output; + const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; + + /* if no room allocated for the error variabels, allocate it now */ + if(ann->train_errors == NULL) + { + ann->train_errors = (fann_type *) calloc(ann->total_neurons, sizeof(fann_type)); + if(ann->train_errors == NULL) + { + fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); + return; + } + } + else + { + /* clear the error variabels */ + memset(ann->train_errors, 0, (ann->total_neurons) * sizeof(fann_type)); + } + error_begin = ann->train_errors; + +#ifdef DEBUGTRAIN + printf("\ncalculate errors\n"); +#endif + /* calculate the error and place it in the output layer */ + error_it = error_begin + (last_layer_begin - first_neuron); + + for(; last_layer_begin != last_layer_end; last_layer_begin++) + { + neuron_value = last_layer_begin->value; + neuron_diff = *desired_output - neuron_value; + + neuron_diff = fann_update_MSE_lw(ann, last_layer_begin, neuron_diff, label_weight); + + if(ann->train_error_function) + { /* TODO make switch when more functions */ + if(neuron_diff < -.9999999) + neuron_diff = -17.0; + else if(neuron_diff > .9999999) + neuron_diff = 17.0; + else + neuron_diff = (fann_type) log((1.0 + neuron_diff) / (1.0 - neuron_diff)); + } + + *error_it = fann_activation_derived(last_layer_begin->activation_function, + last_layer_begin->activation_steepness, neuron_value, + last_layer_begin->sum) * neuron_diff * label_weight; + + desired_output++; + error_it++; + + ann->num_MSE++; + } +} + + + /* INTERNAL FUNCTION Propagate the error backwards from the output layer. @@ -364,6 +532,93 @@ void fann_backpropagate_MSE(struct fann *ann) } } +void fann_backpropagate_MSE_firstlayer(struct fann *ann) +{ + fann_type tmp_error; + unsigned int i; + struct fann_layer *layer_it; + struct fann_neuron *neuron_it, *last_neuron; + struct fann_neuron **connections; + + fann_type *error_begin = ann->train_errors; + fann_type *error_prev_layer; + fann_type *weights; + const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; + const struct fann_layer *second_layer = ann->first_layer + 1; + const struct fann_layer *first_layer = ann->first_layer; + struct fann_layer *last_layer = ann->last_layer; + + /* go through all the layers, from last to first. + * And propagate the error backwards */ + for(layer_it = last_layer - 1; layer_it > first_layer; --layer_it) + { + last_neuron = layer_it->last_neuron; + + /* for each connection in this layer, propagate the error backwards */ + if(ann->connection_rate >= 1) + { + if(ann->network_type == FANN_NETTYPE_LAYER) + { + error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron); + } + else + { + error_prev_layer = error_begin; + } + + for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) + { + + tmp_error = error_begin[neuron_it - first_neuron]; + weights = ann->weights + neuron_it->first_con; + for(i = neuron_it->last_con - neuron_it->first_con; i--;) + { + /*printf("i = %d\n", i); + * printf("error_prev_layer[%d] = %f\n", i, error_prev_layer[i]); + * printf("weights[%d] = %f\n", i, weights[i]); */ + error_prev_layer[i] += tmp_error * weights[i]; + } + } + } + else + { + for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) + { + + tmp_error = error_begin[neuron_it - first_neuron]; + weights = ann->weights + neuron_it->first_con; + connections = ann->connections + neuron_it->first_con; + for(i = neuron_it->last_con - neuron_it->first_con; i--;) + { + error_begin[connections[i] - first_neuron] += tmp_error * weights[i]; + } + } + } + + /* then calculate the actual errors in the previous layer */ + error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron); + last_neuron = (layer_it - 1)->last_neuron; + + if(layer_it != second_layer){ + for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++) + { + *error_prev_layer *= fann_activation_derived(neuron_it->activation_function, + neuron_it->activation_steepness, neuron_it->value, neuron_it->sum); + error_prev_layer++; + } + } else { + //for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++) + //{ + // printf("\n%f %f %d\n", neuron_it->value, neuron_it->activation_steepness, neuron_it->activation_function); + //} + + } + + } +} + + + /* INTERNAL FUNCTION Update weights for incremental training */ @@ -729,7 +984,7 @@ void fann_update_weights_irpropm(struct fann *ann, unsigned int first_weight, un for(; i != past_end; i++) { prev_step = fann_max(prev_steps[i], (fann_type) 0.0001); /* prev_step may not be zero because then the training will stop */ - slope = train_slopes[i]; + slope = train_slopes[i] - ann->learning_l2_norm * weights[i]; prev_slope = prev_train_slopes[i]; same_sign = prev_slope * slope; @@ -748,7 +1003,7 @@ void fann_update_weights_irpropm(struct fann *ann, unsigned int first_weight, un if(weights[i] < -1500) weights[i] = -1500; } - else + else if(slope > 0) //if no error, no update sign(0) = 0 { weights[i] += next_step; if(weights[i] > 1500) @@ -1045,3 +1300,5 @@ FANN_GET_SET(float, sarprop_temperature) FANN_GET_SET(enum fann_stopfunc_enum, train_stop_function) FANN_GET_SET(fann_type, bit_fail_limit) FANN_GET_SET(float, learning_momentum) +FANN_GET_SET(float, learning_l2_norm) + diff --git a/src/fann_train_data.c b/src/fann_train_data.c index fd706730..7b2b7457 100644 --- a/src/fann_train_data.c +++ b/src/fann_train_data.c @@ -152,6 +152,58 @@ float fann_train_epoch_irpropm(struct fann *ann, struct fann_train_data *data) return fann_get_MSE(ann); } +float fann_train_epoch_irpropm_gradient(struct fann *ann, struct fann_train_data *data, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*), void* errorFuncdata) +{ + unsigned int i; + + if(ann->prev_train_slopes == NULL) + { + fann_clear_train_arrays(ann); + } + + fann_reset_MSE(ann); + + for(i = 0; i < data->num_data; i++) + { + fann_run(ann, data->input[i]); + fann_compute_MSE_gradient(ann, data->input[i], errorFunction, errorFuncdata); + fann_backpropagate_MSE(ann); + fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); + } + + fann_update_weights_irpropm(ann, 0, ann->total_connections); + + return fann_get_MSE(ann); +} + + + +float fann_train_epoch_irpropm_lw(struct fann *ann, struct fann_train_data *data, fann_type* label_weight) +{ + unsigned int i; + + if(ann->prev_train_slopes == NULL) + { + fann_clear_train_arrays(ann); + } + + fann_reset_MSE(ann); + + for(i = 0; i < data->num_data; i++) + { + fann_run(ann, data->input[i]); + fann_compute_MSE_lw(ann, data->output[i], label_weight[i]); + fann_backpropagate_MSE(ann); + fann_update_slopes_batch(ann, ann->first_layer + 1, ann->last_layer - 1); + } + + fann_update_weights_irpropm(ann, 0, ann->total_connections); + + return fann_get_MSE(ann); +} + + + /* * Internal train function */ @@ -244,6 +296,24 @@ FANN_EXTERNAL float FANN_API fann_train_epoch(struct fann *ann, struct fann_trai return 0; } +FANN_EXTERNAL float FANN_API fann_train_epoch_lw(struct fann *ann, struct fann_train_data *data, +fann_type* label_weight) +{ + if(fann_check_input_output_sizes(ann, data) == -1) + return 0; + + switch (ann->training_algorithm) + { + case FANN_TRAIN_RPROP: + return fann_train_epoch_irpropm_lw(ann, data, label_weight); + default: + printf("FANN : fann_train_epoch_lw not implemented with others algo\n"); + } + return 0; +} + + + FANN_EXTERNAL void FANN_API fann_train_on_data(struct fann *ann, struct fann_train_data *data, unsigned int max_epochs, unsigned int epochs_between_reports, diff --git a/src/include/fann.h b/src/include/fann.h index 633aa027..ef027c71 100644 --- a/src/include/fann.h +++ b/src/include/fann.h @@ -493,6 +493,10 @@ FANN_EXTERNAL void FANN_API fann_get_bias_array(struct fann *ann, unsigned int * FANN_EXTERNAL void FANN_API fann_get_connection_array(struct fann *ann, struct fann_connection *connections); +FANN_EXTERNAL fann_type FANN_API fann_get_l1_norm(struct fann *ann); + +FANN_EXTERNAL fann_type FANN_API fann_get_l2_norm(struct fann *ann); + /* Function: fann_set_weight_array Set connections in the network. diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 3ab2193a..9c2183f7 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -46,6 +46,10 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #define fann_sigmoid_symmetric_real(sum) (2.0f/(1.0f + exp(-2.0f * sum)) - 1.0f) #define fann_sigmoid_symmetric_derive(steepness, value) steepness * (1.0f - (value*value)) +/* FANN_SIGMOID_SYMMETRIC_LECUN */ +#define fann_sigmoid_symmetric_lecun_real(sum) sqrt(3.f) * (2.0f/(1.0f + exp(-2.0f * sum)) - 1.0f) +#define fann_sigmoid_symmetric_lecun_derive(steepness, value) steepness * ( sqrt(3.f) - (value*value)/sqrt(3.f) ) + /* FANN_GAUSSIAN */ /* #define fann_gaussian(steepness, sum) (exp(-sum * steepness * sum * steepness)) */ #define fann_gaussian_real(sum) (exp(-sum * sum)) @@ -82,6 +86,13 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA #define fann_cos_real(sum) (cos(sum)/2.0f+0.5f) #define fann_cos_derive(steepness, sum) (steepness*-sin(steepness*sum)/2.0f) +/* FANN_RELU */ +#define fann_relu_real(sum) (sum > 0.0 ? sum : 0.0) +#define fann_relu_derive(steepness, sum) (sum > 0.0 ? steepness : 0.0) + +#define fann_leaky_relu_real(sum) (sum > 0.0 ? sum : (sum / 100.0)) +#define fann_leaky_relu_derive(steepness, sum) (sum > 0.0 ? steepness : (steepness / 100.0)) + #define fann_activation_switch(activation_function, value, result) \ switch(activation_function) \ { \ @@ -97,6 +108,9 @@ switch(activation_function) \ case FANN_SIGMOID: \ result = (fann_type)fann_sigmoid_real(value); \ break; \ + case FANN_SIGMOID_SYMMETRIC_LECUN: \ + result = (fann_type)fann_sigmoid_symmetric_lecun_real(value); \ + break; \ case FANN_SIGMOID_SYMMETRIC: \ result = (fann_type)fann_sigmoid_symmetric_real(value); \ break; \ @@ -139,6 +153,12 @@ switch(activation_function) \ case FANN_GAUSSIAN_STEPWISE: \ result = 0; \ break; \ + case FANN_RELU: \ + result = (fann_type)fann_relu_real(value); \ + break; \ + case FANN_LEAKY_RELU: \ + result = (fann_type)fann_leaky_relu_real(value); \ + break; \ } #endif diff --git a/src/include/fann_cpp.h b/src/include/fann_cpp.h index 6f54b34a..3fb69601 100644 --- a/src/include/fann_cpp.h +++ b/src/include/fann_cpp.h @@ -1950,6 +1950,23 @@ namespace FANN { } } + float get_learning_l2_norm() + { + float learning_l2_norm = 0.0f; + if (ann != NULL) + { + learning_l2_norm = fann_get_learning_l2_norm(ann); + } + return learning_l2_norm; + } + + void set_learning_l2_norm(float learning_l2_norm) + { + if (ann != NULL) + { + fann_set_learning_l2_norm(ann, learning_l2_norm); + } + } /* Method: get_train_stop_function Returns the the stop function used during training. diff --git a/src/include/fann_data.h b/src/include/fann_data.h index 99f42c76..c9ecb4ba 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -226,7 +226,10 @@ enum fann_activationfunc_enum FANN_SIN_SYMMETRIC, FANN_COS_SYMMETRIC, FANN_SIN, - FANN_COS + FANN_COS, + FANN_SIGMOID_SYMMETRIC_LECUN, + FANN_RELU, + FANN_LEAKY_RELU }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -258,7 +261,10 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = { "FANN_SIN_SYMMETRIC", "FANN_COS_SYMMETRIC", "FANN_SIN", - "FANN_COS" + "FANN_COS", + "FANN_SIGMOID_SYMMETRIC_LECUN", + "FANN_RELU", + "FANN_LEAKY_RELU" }; /* Enum: fann_errorfunc_enum @@ -491,6 +497,7 @@ struct fann /* The learning momentum used for backpropagation algorithm. */ float learning_momentum; + float learning_l2_norm; /* the connection rate of the network * between 0 and 1, 1 meaning fully connected diff --git a/src/include/fann_train.h b/src/include/fann_train.h index ae377d96..d5c24e08 100644 --- a/src/include/fann_train.h +++ b/src/include/fann_train.h @@ -212,6 +212,10 @@ FANN_EXTERNAL void FANN_API fann_train_on_file(struct fann *ann, const char *fil This function appears in FANN >= 1.2.0. */ FANN_EXTERNAL float FANN_API fann_train_epoch(struct fann *ann, struct fann_train_data *data); +FANN_EXTERNAL float FANN_API fann_train_epoch_lw(struct fann *ann, struct fann_train_data *data, fann_type* label_weight); +FANN_EXTERNAL float FANN_API fann_train_epoch_irpropm_gradient(struct fann *ann, struct fann_train_data *data, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*),void*); +FANN_EXTERNAL void FANN_API fann_compute_MSE_gradient(struct fann *, fann_type *, fann_type (*errorFunction)(fann_type*,fann_type*,int,void*), void*); +FANN_EXTERNAL void FANN_API fann_backpropagate_MSE_firstlayer(struct fann *); #endif /* NOT FIXEDFANN */ /* Function: fann_test_data @@ -828,7 +832,8 @@ FANN_EXTERNAL float FANN_API fann_get_learning_momentum(struct fann *ann); This function appears in FANN >= 2.0.0. */ FANN_EXTERNAL void FANN_API fann_set_learning_momentum(struct fann *ann, float learning_momentum); - +FANN_EXTERNAL float FANN_API fann_get_learning_l2_norm(struct fann *ann); +FANN_EXTERNAL void FANN_API fann_set_learning_l2_norm(struct fann *ann, float learning_l2_norm); /* Function: fann_get_activation_function