From 46fcb2885a61169149ee13e42670617278858426 Mon Sep 17 00:00:00 2001 From: Jeremy Sadler <53983960+jezsadler@users.noreply.github.com> Date: Sat, 11 Nov 2023 01:29:13 +0000 Subject: [PATCH 1/2] Adding support for explicit padding in 2D layers --- src/omlt/io/onnx_parser.py | 22 +++++++++++----------- src/omlt/neuralnet/layer.py | 33 ++++++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/src/omlt/io/onnx_parser.py b/src/omlt/io/onnx_parser.py index 511261c0..713db274 100644 --- a/src/omlt/io/onnx_parser.py +++ b/src/omlt/io/onnx_parser.py @@ -368,16 +368,17 @@ def _consume_conv_nodes(self, node, next_nodes): raise ValueError( f"{node} has multiple groups ({attr['group']}). This is not supported." ) - if "pads" in attr and np.any(attr["pads"]): - raise ValueError( - f"{node} has non-zero pads ({attr['pads']}). This is not supported." - ) + if "pads" in attr: + pads = attr["pads"] + else: + pads = None # generate new nodes for the node output - padding = 0 + + padding = [pads[i] + pads[i + len(node.input)] for i in range(len(node.input))] output_size = [out_channels] - for w, k, s in zip(input_output_size[1:], kernel_shape, strides): - new_w = int((w - k + 2 * padding) / s) + 1 + for w, k, s, p in zip(input_output_size[1:], kernel_shape, strides, padding): + new_w = int((w - k + p) / s) + 1 output_size.append(new_w) activation = "linear" @@ -401,6 +402,7 @@ def _consume_conv_nodes(self, node, next_nodes): output_size, strides, weights, + pads=pads, activation=activation, input_index_mapper=transformer, ) @@ -467,6 +469,7 @@ def _consume_pool_nodes(self, node, next_nodes): kernel_depth = attr["kernel_shape"][0] kernel_shape = attr["kernel_shape"][1:] strides = attr["strides"] if "strides" in attr else [1] * len(kernel_shape) + pads = attr["pads"] if "pads" in attr else None # check only kernel shape, stride, storage order are set # everything else is not supported @@ -474,10 +477,6 @@ def _consume_pool_nodes(self, node, next_nodes): raise ValueError( f"{node.name} has non-identity dilations ({attr['dilations']}). This is not supported." ) - if "pads" in attr and np.any(attr["pads"]): - raise ValueError( - f"{node.name} has non-zero pads ({attr['pads']}). This is not supported." - ) if ("auto_pad" in attr) and (attr["auto_pad"] != "NOTSET"): raise ValueError( f"{node.name} has autopad set ({attr['auto_pad']}). This is not supported." @@ -519,6 +518,7 @@ def _consume_pool_nodes(self, node, next_nodes): pool_func_name, tuple(kernel_shape), kernel_depth, + pads=pads, activation=activation, input_index_mapper=transformer, ) diff --git a/src/omlt/neuralnet/layer.py b/src/omlt/neuralnet/layer.py index d7f7fa89..956da4bf 100644 --- a/src/omlt/neuralnet/layer.py +++ b/src/omlt/neuralnet/layer.py @@ -225,6 +225,8 @@ class Layer2D(Layer): the size of the output. strides : matrix-like stride of the kernel. + pads : matrix-like + Padding for the kernel. Given as [left, bottom, right, top] activation : str or None activation function name input_index_mapper : IndexMapper or None @@ -237,6 +239,7 @@ def __init__( output_size, strides, *, + pads=None, activation=None, input_index_mapper=None, ): @@ -247,12 +250,21 @@ def __init__( input_index_mapper=input_index_mapper, ) self.__strides = strides + if pads is None: + self.__pads = [0, 0, 0, 0] + else: + self.__pads = pads @property def strides(self): """Return the stride of the layer""" return self.__strides + @property + def pads(self): + """Return the padding of the layer""" + return self.__pads + @property def kernel_shape(self): """Return the shape of the kernel""" @@ -280,12 +292,14 @@ def kernel_index_with_input_indexes(self, out_d, out_r, out_c): kernel_d = self.kernel_depth [kernel_r, kernel_c] = self.kernel_shape [rows_stride, cols_stride] = self.__strides + [pads_row, pads_col] = self.__pads[:1] start_in_d = 0 - start_in_r = out_r * rows_stride - start_in_c = out_c * cols_stride - mapper = lambda x: x - if self.input_index_mapper is not None: - mapper = self.input_index_mapper + start_in_r = out_r * rows_stride - pads_row + start_in_c = out_c * cols_stride - pads_col + # Defined but never used: + # mapper = lambda x: x + # if self.input_index_mapper is not None: + # mapper = self.input_index_mapper for k_d in range(kernel_d): for k_r in range(kernel_r): @@ -299,6 +313,7 @@ def kernel_index_with_input_indexes(self, out_d, out_r, out_c): # even though we loop over ALL kernel indexes. if not all( input_index[i] < self.input_size[i] + and input_index[i] >= 0 for i in range(len(input_index)) ): continue @@ -345,6 +360,8 @@ class PoolingLayer2D(Layer2D): the size of the output. strides : matrix-like stride of the kernel. + pads : matrix-like + Padding for the kernel. Given as [left, bottom, right, top] pool_func : str name of function used to pool values in a kernel to a single value. transpose : bool @@ -367,6 +384,7 @@ def __init__( kernel_shape, kernel_depth, *, + pads=None, activation=None, input_index_mapper=None, ): @@ -374,6 +392,7 @@ def __init__( input_size, output_size, strides, + pads=pads, activation=activation, input_index_mapper=input_index_mapper, ) @@ -421,6 +440,8 @@ class ConvLayer2D(Layer2D): stride of the cross-correlation kernel. kernel : matrix-like the cross-correlation kernel. + pads : matrix-like + Padding for the kernel. Given as [left, bottom, right, top] activation : str or None activation function name input_index_mapper : IndexMapper or None @@ -434,6 +455,7 @@ def __init__( strides, kernel, *, + pads=None, activation=None, input_index_mapper=None, ): @@ -441,6 +463,7 @@ def __init__( input_size, output_size, strides, + pads=pads, activation=activation, input_index_mapper=input_index_mapper, ) From abf7a4dc1c68e1867adef011a75a6968957e72ee Mon Sep 17 00:00:00 2001 From: Jeremy Sadler <53983960+jezsadler@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:19:34 +0000 Subject: [PATCH 2/2] Fixed parsing of padded conv layers. --- src/omlt/io/onnx_parser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/omlt/io/onnx_parser.py b/src/omlt/io/onnx_parser.py index 713db274..5a15b8b0 100644 --- a/src/omlt/io/onnx_parser.py +++ b/src/omlt/io/onnx_parser.py @@ -371,11 +371,12 @@ def _consume_conv_nodes(self, node, next_nodes): if "pads" in attr: pads = attr["pads"] else: - pads = None + pads = 2*(len(input_output_size)-1)*[0] # generate new nodes for the node output - - padding = [pads[i] + pads[i + len(node.input)] for i in range(len(node.input))] + padding = [ + pads[i] + pads[i + len(input_output_size)-1] + for i in range(len(input_output_size)-1)] output_size = [out_channels] for w, k, s, p in zip(input_output_size[1:], kernel_shape, strides, padding): new_w = int((w - k + p) / s) + 1