diff --git a/src/fann.c b/src/fann.c index b8e3b0a..f225ab4 100644 --- a/src/fann.c +++ b/src/fann.c @@ -692,12 +692,12 @@ FANN_EXTERNAL fann_type *FANN_API fann_run(struct fann *ann, fann_type *input) { : (neuron_sum > multiplier) ? multiplier : neuron_sum); break; - case FANN_LINEAR_PIECE_LEAKY: - neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum); - break; case FANN_LINEAR_PIECE_RECT: neuron_it->value = (fann_type)((neuron_sum < 0) ? 0 : neuron_sum); break; + case FANN_LINEAR_PIECE_RECT_LEAKY: + neuron_it->value = (fann_type)((neuron_sum < 0) ? 0.01 * neuron_sum : neuron_sum); + break; case FANN_ELLIOT: case FANN_ELLIOT_SYMMETRIC: case FANN_GAUSSIAN: diff --git a/src/fann_cascade.c b/src/fann_cascade.c index 9dea5a2..0935a4f 100644 --- a/src/fann_cascade.c +++ b/src/fann_cascade.c @@ -680,8 +680,8 @@ fann_type fann_train_candidates_epoch(struct fann *ann, struct fann_train_data * case FANN_GAUSSIAN_STEPWISE: case FANN_ELLIOT: case FANN_LINEAR_PIECE: - case FANN_LINEAR_PIECE_LEAKY: case FANN_LINEAR_PIECE_RECT: + case FANN_LINEAR_PIECE_RECT_LEAKY: case FANN_SIN: case FANN_COS: break; diff --git a/src/fann_train.c b/src/fann_train.c index 786eb97..8f974bd 100644 --- a/src/fann_train.c +++ b/src/fann_train.c @@ -40,10 +40,10 @@ fann_type fann_activation_derived(unsigned int activation_function, fann_type st case FANN_LINEAR_PIECE: case FANN_LINEAR_PIECE_SYMMETRIC: return (fann_type)fann_linear_derive(steepness, value); - case FANN_LINEAR_PIECE_LEAKY: - return (fann_type)((value < 0) ? steepness * 0.01 : steepness); case FANN_LINEAR_PIECE_RECT: return (fann_type)((value < 0) ? 0 : steepness); + case FANN_LINEAR_PIECE_RECT_LEAKY: + return (fann_type)((value < 0) ? steepness * 0.01 : steepness); case FANN_SIGMOID: case FANN_SIGMOID_STEPWISE: value = fann_clip(value, 0.01f, 0.99f); @@ -129,8 +129,8 @@ fann_type fann_update_MSE(struct fann *ann, struct fann_neuron *neuron, fann_typ case FANN_GAUSSIAN_STEPWISE: case FANN_ELLIOT: case FANN_LINEAR_PIECE: - case FANN_LINEAR_PIECE_LEAKY: case FANN_LINEAR_PIECE_RECT: + case FANN_LINEAR_PIECE_RECT_LEAKY: case FANN_SIN: case FANN_COS: break; diff --git a/src/include/fann_activation.h b/src/include/fann_activation.h index 6e99d68..1de138c 100644 --- a/src/include/fann_activation.h +++ b/src/include/fann_activation.h @@ -178,12 +178,12 @@ __doublefann_h__ is not defined case FANN_GAUSSIAN_STEPWISE: \ result = 0; \ break; \ - case FANN_LINEAR_PIECE_LEAKY: \ - result = (fann_type)((value < 0) ? value * 0.01 : value); \ - break; \ case FANN_LINEAR_PIECE_RECT: \ result = (fann_type)((value < 0) ? 0 : value); \ break; \ + case FANN_LINEAR_PIECE_RECT_LEAKY: \ + result = (fann_type)((value < 0) ? value * 0.01 : value); \ + break; \ } #endif diff --git a/src/include/fann_data.h b/src/include/fann_data.h index a5dc91a..f9040c2 100644 --- a/src/include/fann_data.h +++ b/src/include/fann_data.h @@ -196,16 +196,16 @@ static char const *const FANN_TRAIN_NAMES[] = {"FANN_TRAIN_INCREMENTAL", "FANN_T * y = cos(x*s)/2+0.5 * d = s*-sin(x*s)/2 - FANN_LINEAR_PIECE_LEAKY - leaky ReLU - * span: -inf < y < inf - * y = x<0? 0.01*x: x - * d = x<0? 0.01: 1 - FANN_LINEAR_PIECE_RECT - ReLU * span: -inf < y < inf * y = x<0? 0: x * d = x<0? 0: 1 + FANN_LINEAR_PIECE_RECT_LEAKY - leaky ReLU + * span: -inf < y < inf + * y = x<0? 0.01*x: x + * d = x<0? 0.01: 1 + See also: , , , , @@ -234,8 +234,8 @@ enum fann_activationfunc_enum { FANN_COS_SYMMETRIC, FANN_SIN, FANN_COS, - FANN_LINEAR_PIECE_LEAKY, - FANN_LINEAR_PIECE_RECT + FANN_LINEAR_PIECE_RECT, + FANN_LINEAR_PIECE_RECT_LEAKY }; /* Constant: FANN_ACTIVATIONFUNC_NAMES @@ -267,8 +267,8 @@ static char const *const FANN_ACTIVATIONFUNC_NAMES[] = {"FANN_LINEAR", "FANN_COS_SYMMETRIC", "FANN_SIN", "FANN_COS", - "FANN_LINEAR_PIECE_LEAKY", - "FANN_LINEAR_PIECE_RECT"}; + "FANN_LINEAR_PIECE_RECT", + "FANN_LINEAR_PIECE_RECT_LEAKY"}; /* Enum: fann_errorfunc_enum Error function used during training. diff --git a/src/include/fann_data_cpp.h b/src/include/fann_data_cpp.h index 766f9b6..2006df2 100644 --- a/src/include/fann_data_cpp.h +++ b/src/include/fann_data_cpp.h @@ -200,16 +200,16 @@ enum training_algorithm_enum { * y = cos(x*s) * d = s*-sin(x*s) - FANN_LINEAR_PIECE_LEAKY - leaky ReLU - * span: -inf < y < inf - * y = x<0? 0.01*x: x - * d = x<0? 0.01: 1 - FANN_LINEAR_PIECE_RECT - ReLU * span: -inf < y < inf * y = x<0? 0: x * d = x<0? 0: 1 + FANN_LINEAR_PIECE_RECT_LEAKY - leaky ReLU + * span: -inf < y < inf + * y = x<0? 0.01*x: x + * d = x<0? 0.01: 1 + See also: , @@ -231,8 +231,8 @@ enum activation_function_enum { LINEAR_PIECE_SYMMETRIC, SIN_SYMMETRIC, COS_SYMMETRIC, - LINEAR_PIECE_LEAKY, - LINEAR_PIECE_RECT + LINEAR_PIECE_RECT, + LINEAR_PIECE_RECT_LEAKY }; /* Enum: network_type_enum diff --git a/tests/fann_test_train.cpp b/tests/fann_test_train.cpp index ee3f83f..167a209 100644 --- a/tests/fann_test_train.cpp +++ b/tests/fann_test_train.cpp @@ -33,6 +33,18 @@ TEST_F(FannTestTrain, TrainOnReLUSimpleXor) { EXPECT_LT(net.test_data(data), 0.001); } +TEST_F(FannTestTrain, TrainOnReLULeakySimpleXor) { + neural_net net(LAYER, 3, 2, 3, 1); + + data.set_train_data(4, 2, xorInput, 1, xorOutput); + net.set_activation_function_hidden(FANN::LINEAR_PIECE_RECT_LEAKY); + net.set_activation_steepness_hidden(1.0); + net.train_on_data(data, 100, 100, 0.001); + + EXPECT_LT(net.get_MSE(), 0.001); + EXPECT_LT(net.test_data(data), 0.001); +} + TEST_F(FannTestTrain, TrainSimpleIncrementalXor) { neural_net net(LAYER, 3, 2, 3, 1);