Skip to content

Commit

Permalink
[CI SKIP] WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@darioizzo
darioizzo committed Nov 1, 2023
1 parent 253450f commit c3ecf67
Show file tree
Hide file tree
Showing 4 changed files with 188 additions and 43 deletions.
7 changes: 7 additions & 0 deletions include/heyoka/kw.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ IGOR_MAKE_NAMED_ARGUMENT(parallel_mode);
IGOR_MAKE_NAMED_ARGUMENT(prec);
IGOR_MAKE_NAMED_ARGUMENT(mu);

// kwargs for the ffnn
IGOR_MAKE_NAMED_ARGUMENT(inputs);
IGOR_MAKE_NAMED_ARGUMENT(nn_hidden);
IGOR_MAKE_NAMED_ARGUMENT(n_out);
IGOR_MAKE_NAMED_ARGUMENT(activations);
IGOR_MAKE_NAMED_ARGUMENT(nn_wb);

} // namespace kw

HEYOKA_END_NAMESPACE
Expand Down
93 changes: 91 additions & 2 deletions include/heyoka/model/ffnn.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,96 @@
#include <vector>

#include <heyoka/config.hpp>
#include <heyoka/detail/igor.hpp>
#include <heyoka/detail/visibility.hpp>
#include <heyoka/expression.hpp>

HEYOKA_BEGIN_NAMESPACE

namespace model
{
namespace detail
{
template <typename... KwArgs>
auto ffnn_common_opts(KwArgs &&...kw_args)
{
igor::parser p{kw_args...};

static_assert(!p.has_unnamed_arguments(), "This function accepts only named arguments");

// Network inputs. Mandatory
auto inputs = [&p]() {
if constexpr (p.has(kw::inputs)) {
return std::vector<expression>{p(kw::inputs)};
} else {
static_assert(::heyoka::detail::always_false_v<KwArgs...>,
"The 'inputs' keyword argument is necessary but it was not provided");
}
}();

// Number of hidden neurons per hidden layer. Mandatory
auto nn_hidden = [&p]() {
if constexpr (p.has(kw::nn_hidden)) {
return std::vector<std::uint32_t>{p(kw::nn_hidden)};
} else {
static_assert(::heyoka::detail::always_false_v<KwArgs...>,
"The 'nn_hidden' keyword argument is necessary but it was not provided");
}
}();

// Number of network outputs. Mandatory
auto n_out = [&p]() {
if constexpr (p.has(kw::n_out)) {
return std::uint32_t{p(kw::n_out)};
} else {
static_assert(::heyoka::detail::always_false_v<KwArgs...>,
"The 'n_out' keyword argument is necessary but it was not provided");
}
}();

// Network activation functions. Mandatory
auto activations = [&p]() {
if constexpr (p.has(kw::activations)) {
return std::vector<std::function<expression(const expression &)>>{p(kw::activations)};
} else {
static_assert(::heyoka::detail::always_false_v<KwArgs...>,
"The 'activations' keyword argument is necessary but it was not provided");
}
}();

// Network weights and biases. Defaults to heyoka parameters.
auto nn_wb = [&p, &nn_hidden, &inputs, n_out]() {
if constexpr (p.has(kw::nn_wb)) {
return std::vector<expression> {p(kw::nn_wb)};
} else {
// Number of hidden layers (defined as all neuronal columns that are nor input nor output neurons)
auto n_hidden_layers = boost::numeric_cast<std::uint32_t>(nn_hidden.size());
// Number of neuronal layers (counting input and output)
auto n_layers = n_hidden_layers + 2;
// Number of inputs
auto n_in = boost::numeric_cast<std::uint32_t>(inputs.size());
// Number of neurons per neuronal layer
std::vector<std::uint32_t> n_neurons = nn_hidden;
n_neurons.insert(n_neurons.begin(), n_in);
n_neurons.insert(n_neurons.end(), n_out);
// Number of network parameters (wb: weights and biases, w: only weights)
std::uint32_t n_wb = 0u;
for (std::uint32_t i = 1u; i < n_layers; ++i) {
n_wb += n_neurons[i - 1] * n_neurons[i];
n_wb += n_neurons[i];
}
std::vector<expression> retval(n_wb);
for (decltype(retval.size()) i = 0; i < retval.size(); ++i) {
retval[i] = heyoka::par[i];
}
return retval;
}
}();

return std::tuple{std::move(inputs), std::move(nn_hidden), std::move(n_out), std::move(activations),
std::move(nn_wb)};
}

// This c++ function returns the symbolic expressions of the `n_out` output neurons in a feed forward neural network,
// as a function of the `n_in` input expressions.
//
Expand All @@ -31,10 +114,16 @@ namespace model
// from the left to right layer of parameters: [W01, W12,W23, ..., B1,B2,B3,....] where the weight matrices Wij are
// to be considered as flattened (row first) and so are the bias vectors.
//
HEYOKA_DLL_PUBLIC std::vector<expression> ffnn_impl(const std::vector<expression> &, std::uint32_t,
const std::vector<std::uint32_t> &,
HEYOKA_DLL_PUBLIC std::vector<expression> ffnn_impl(const std::vector<expression> &, const std::vector<std::uint32_t> &,
std::uint32_t,
const std::vector<std::function<expression(const expression &)>> &,
const std::vector<expression> &);
} // namespace detail

inline constexpr auto ffnn = [](const auto &...kw_args) -> std::vector<expression> {
return std::apply(detail::ffnn_impl, detail::ffnn_common_opts(kw_args...));
};

} // namespace model

HEYOKA_END_NAMESPACE
Expand Down
59 changes: 22 additions & 37 deletions src/model/ffnn.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,81 +20,67 @@

HEYOKA_BEGIN_NAMESPACE

namespace model
{
namespace detail
namespace model::detail
{
std::vector<expression> compute_layer(std::uint32_t layer_id, const std::vector<expression> &inputs,
const std::vector<std::uint32_t> &n_neurons,
const std::function<expression(const expression &)> &activation,
const std::vector<expression> &net_wb, std::uint32_t n_net_w,
const std::vector<expression> &nn_wb, std::uint32_t n_net_w,
std::uint32_t &wcounter, std::uint32_t &bcounter)
{
assert(layer_id > 0);
auto n_neurons_prev_layer = boost::numeric_cast<std::uint32_t>(inputs.size());
auto n_neurons_curr_layer = n_neurons[layer_id];

std::vector<expression> retval(n_neurons_curr_layer, 0_dbl);
fmt::print("net_wb: {}\n", net_wb.size());
std::cout << std::endl;

for (std::uint32_t i = 0u; i < n_neurons_curr_layer; ++i) {
for (std::uint32_t j = 0u; j < n_neurons_prev_layer; ++j) {
fmt::print("layer, i, j, idx: {}, {}, {}, {}\n", layer_id, i, j, wcounter);
std::cout << std::endl;

// Add the weight and update the weight counter
retval[i] += net_wb[wcounter] * inputs[j];
retval[i] += nn_wb[wcounter] * inputs[j];
++wcounter;
}
fmt::print("idxb {}\n", bcounter + n_net_w);
std::cout << std::endl;

// Add the bias and update the counter
retval[i] += net_wb[bcounter + n_net_w];
retval[i] += nn_wb[bcounter + n_net_w];
++bcounter;
// Activation function
retval[i] = activation(retval[i]);
}
return retval;
}
} // namespace detail

std::vector<expression> ffnn_impl(
// NOLINTNEXTLINE(bugprone-easily-swappable-parameters)
const std::vector<expression> &in, std::uint32_t n_out,
const std::vector<std::uint32_t> &n_neurons_per_hidden_layer,
const std::vector<std::function<expression(const expression &)>> &activations,
const std::vector<expression> &net_wb)
std::vector<expression> ffnn_impl(const std::vector<expression> &in, const std::vector<std::uint32_t> &nn_hidden,
std::uint32_t n_out,
const std::vector<std::function<expression(const expression &)>> &activations,
const std::vector<expression> &nn_wb)
{
// Sanity checks
if (n_neurons_per_hidden_layer.size() + 1 != activations.size()) {
if (nn_hidden.size() + 1 != activations.size()) {
throw std::invalid_argument(fmt::format(
"The number of hidden layers, as detected from the inputs, was {}, while"
"The number of hidden layers, as detected from the inputs, was {}, while "
"the number of activation function supplied was {}. A FFNN needs exactly one more activation function "
"than the number of hidden layers.",
n_neurons_per_hidden_layer.size(), activations.size()));
nn_hidden.size(), activations.size()));
}
if (in.empty()) {
throw std::invalid_argument("The inputs provided to the ffnn seem to be an empty vector.");
throw std::invalid_argument("The inputs provided to the FFNN is an empty vector.");
}
if (n_out == 0) {
throw std::invalid_argument("The number of network outputs cannot be zero.");
}
if (!std::all_of(n_neurons_per_hidden_layer.begin(), n_neurons_per_hidden_layer.end(),
[](std::uint32_t item) { return item > 0; })) {
if (!std::all_of(nn_hidden.begin(), nn_hidden.end(), [](std::uint32_t item) { return item > 0; })) {
throw std::invalid_argument("The number of neurons for each hidden layer must be greater than zero!");
}
if (n_neurons_per_hidden_layer.empty()) { // TODO(darioizzo): maybe this is actually a wanted corner case, remove?
throw std::invalid_argument("The number of hidden layers cannot be zero.");
}

// Number of hidden layers (defined as all neuronal columns that are nor input nor output neurons)
auto n_hidden_layers = boost::numeric_cast<std::uint32_t>(n_neurons_per_hidden_layer.size());
auto n_hidden_layers = boost::numeric_cast<std::uint32_t>(nn_hidden.size());
// Number of neuronal layers (counting input and output)
auto n_layers = n_hidden_layers + 2;
// Number of inputs
auto n_in = boost::numeric_cast<std::uint32_t>(in.size());
// Number of neurons per neuronal layer
std::vector<std::uint32_t> n_neurons = n_neurons_per_hidden_layer;
std::vector<std::uint32_t> n_neurons = nn_hidden;
n_neurons.insert(n_neurons.begin(), n_in);
n_neurons.insert(n_neurons.end(), n_out);
// Number of network parameters (wb: weights and biases, w: only weights)
Expand All @@ -106,20 +92,19 @@ std::vector<expression> ffnn_impl(
n_net_wb += n_neurons[i];
}
// Sanity check
if (net_wb.size() != n_net_wb) {
if (nn_wb.size() != n_net_wb) {
throw std::invalid_argument(fmt::format(
"The number of network parameters, detected from its structure to be {}, does not match the size of"
"the corresponding expressions {} ",
n_net_wb, net_wb.size()));
"The number of network parameters, detected from its structure to be {}, does not match the size of "
"the corresponding expressions: {}.",
n_net_wb, nn_wb.size()));
}

// Now we build the expressions recursively transvering from layer to layer (L = f(Wx+b)))

std::vector<expression> retval = in;
std::uint32_t wcounter = 0;
std::uint32_t bcounter = 0;
for (std::uint32_t i = 1u; i < n_layers; ++i) {
retval = detail::compute_layer(i, retval, n_neurons, activations[i - 1], net_wb, n_net_w, wcounter, bcounter);
retval = detail::compute_layer(i, retval, n_neurons, activations[i - 1], nn_wb, n_net_w, wcounter, bcounter);
}
return retval;
}
Expand Down
72 changes: 68 additions & 4 deletions test/model_ffnn.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,78 @@
#include <heyoka/model/ffnn.hpp>

#include "catch.hpp"
#include "heyoka/kw.hpp"

using namespace heyoka;

TEST_CASE("impl")
{
// A linear layer, just because
auto linear = [](expression ret) -> expression { return ret; };
auto [x] = make_vars("x");
auto my_net = model::ffnn_impl({x}, 2, {2, 2}, {heyoka::tanh, heyoka::tanh, linear},
{1_dbl, 2_dbl, 3_dbl, 4_dbl, 5_dbl, 6_dbl, 7_dbl, 8_dbl, 9_dbl, 0_dbl, 1_dbl, 2_dbl,
3_dbl, 4_dbl, 5_dbl, 6_dbl});
// We also define a few symbols
auto [x, y, z] = make_vars("x", "y", "z");

// First, we test malformed cases and their throws.
// 1 - number of activations function is wrong
REQUIRE_THROWS_AS(model::detail::ffnn_impl({x}, {1}, 2, {heyoka::tanh, heyoka::tanh, linear},
{1_dbl, 2_dbl, 3_dbl, 4_dbl, 5_dbl, 6_dbl}),
std::invalid_argument);
// 2 - number of inputs is zero
REQUIRE_THROWS_AS(
model::detail::ffnn_impl({}, {1}, 2, {heyoka::tanh, heyoka::tanh}, {1_dbl, 2_dbl, 3_dbl, 4_dbl, 5_dbl, 6_dbl}),
std::invalid_argument);
// 3 - number of outputs is zero
REQUIRE_THROWS_AS(model::detail::ffnn_impl({x}, {1}, 0, {heyoka::tanh, heyoka::tanh}, {1_dbl, 2_dbl, 3_dbl, 4_dbl}),
std::invalid_argument);
// 4 - One of the hidden layers has zero neurons
REQUIRE_THROWS_AS(model::detail::ffnn_impl({x}, {1, 0}, 2, {heyoka::tanh, heyoka::tanh, linear}, {1_dbl, 2_dbl}),
std::invalid_argument);
// 5 - Wrong number of weights/biases
REQUIRE_THROWS_AS(
model::detail::ffnn_impl({x}, {1}, 1, {heyoka::tanh, heyoka::tanh}, {1_dbl, 2_dbl, 3_dbl, 5_dbl, 6_dbl}),
std::invalid_argument);

// We now check some hand coded networks
{
auto my_net = model::detail::ffnn_impl({x}, {}, 1, {linear}, {1_dbl, 2_dbl});
REQUIRE(my_net[0] == expression(2_dbl + x));
}
{
auto my_net = model::detail::ffnn_impl({x}, {}, 1, {heyoka::tanh}, {1_dbl, 2_dbl});
REQUIRE(my_net[0] == expression(heyoka::tanh(2_dbl + x)));
}
{
auto my_net = model::detail::ffnn_impl({x}, {1}, 1, {heyoka::tanh, linear}, {1_dbl, 2_dbl, 3_dbl, 4_dbl});
REQUIRE(my_net[0] == expression(4_dbl + (2_dbl * heyoka::tanh(3_dbl + x))));
}
{
auto my_net = model::detail::ffnn_impl({x}, {1}, 1, {heyoka::tanh, heyoka::sin}, {1_dbl, 2_dbl, 3_dbl, 4_dbl});
REQUIRE(my_net[0] == expression(heyoka::sin(4_dbl + (2_dbl * heyoka::tanh(3_dbl + x)))));
}
{
auto my_net = model::detail::ffnn_impl({x, y}, {2}, 1, {heyoka::sin, heyoka::cos},
{1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl});
REQUIRE(my_net[0] == expression(heyoka::cos(1_dbl + 2_dbl * heyoka::sin(1_dbl + x + y))));
}
}

TEST_CASE("igor_iface")
{
auto [x, y, z] = make_vars("x", "y", "z");
{
auto igor_v = model::ffnn(kw::inputs = {x, y}, kw::nn_hidden = std::vector<std::uint32_t>{2u}, kw::n_out = 1u,
kw::activations = std::vector<std::function<expression(const expression &)>>{heyoka::sin, heyoka::cos},
kw::nn_wb = {1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl});
auto vanilla_v = model::detail::ffnn_impl({x, y}, {2u}, 1u, {heyoka::sin, heyoka::cos},
{1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl, 1_dbl});
REQUIRE(igor_v == vanilla_v);
}
// We test the expected setting for the default weights+biases expressions to par[i].
{
auto igor_v = model::ffnn(kw::inputs = {x, y}, kw::nn_hidden = std::vector<std::uint32_t>{2u}, kw::n_out = 1u,
kw::activations = std::vector<std::function<expression(const expression &)>>{heyoka::sin, heyoka::cos});
auto vanilla_v = model::detail::ffnn_impl({x, y}, {2u}, 1u, {heyoka::sin, heyoka::cos},
{par[0], par[1], par[2], par[3], par[4], par[5], par[6], par[7], par[8]});
REQUIRE(igor_v == vanilla_v);
}
}

0 comments on commit c3ecf67

Please sign in to comment.