From 65e5a7b02f674e6ed6e50784a6cfb43a131d2826 Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Tue, 5 Nov 2024 12:58:40 +0100 Subject: [PATCH 1/8] support 1D conv --- main_stream_co copy.py | 69 ++++++++++++ outputs/custom_ssm.onnx | Bin 0 -> 8163 bytes .../mapping/tpu_like_quad_core copy.yaml | 55 ++++++++++ stream/node_tensor.py | 2 +- stream/onnx_utils.py | 26 +++++ stream/parser/onnx/asymmetric_simd.py | 18 +-- stream/parser/onnx/conv.py | 103 ++++++++---------- stream/parser/onnx/einsum.py | 92 ++++++++++++++++ stream/parser/onnx/model.py | 2 +- stream/parser/onnx/operator_parser.py | 8 +- stream/parser/onnx/reduce_1d.py | 2 +- stream/parser/onnx/simd.py | 2 +- stream/parser/onnx/softmax.py | 2 +- stream/utils.py | 21 ---- 14 files changed, 305 insertions(+), 97 deletions(-) create mode 100644 main_stream_co copy.py create mode 100644 outputs/custom_ssm.onnx create mode 100644 stream/inputs/examples/mapping/tpu_like_quad_core copy.yaml create mode 100644 stream/onnx_utils.py create mode 100644 stream/parser/onnx/einsum.py diff --git a/main_stream_co copy.py b/main_stream_co copy.py new file mode 100644 index 00000000..dff081c1 --- /dev/null +++ b/main_stream_co copy.py @@ -0,0 +1,69 @@ +import logging as _logging +import re + +from stream.api import optimize_allocation_co +from stream.utils import CostModelEvaluationLUT +from stream.visualization.memory_usage import plot_memory_usage +from stream.visualization.schedule import ( + visualize_timeline_plotly, +) + +_logging_level = _logging.INFO +_logging_format = "%(asctime)s - %(name)s.%(funcName)s +%(lineno)s - %(levelname)s - %(message)s" +_logging.basicConfig(level=_logging_level, format=_logging_format) + +############################################INPUTS############################################ +accelerator = "stream/inputs/examples/hardware/tpu_like_quad_core.yaml" +workload_path = "outputs/custom_ssm.onnx" +mapping_path = "stream/inputs/examples/mapping/tpu_like_quad_core copy.yaml" +mode = "fused" +layer_stacks = [tuple(range(0, 11)), tuple(range(11, 22))] + list((i,) for i in range(22, 49)) +############################################################################################## + +################################PARSING############################### +hw_name = accelerator.split("/")[-1].split(".")[0] +wl_name = re.split(r"/|\.", workload_path)[-1] +if wl_name == "onnx": + wl_name = re.split(r"/|\.", workload_path)[-2] +experiment_id = f"{hw_name}-{wl_name}-{mode}-constraint_optimization" +###################################################################### + +scme = optimize_allocation_co( + hardware=accelerator, + workload=workload_path, + mapping=mapping_path, + mode=mode, + layer_stacks=layer_stacks, + experiment_id=experiment_id, + output_path="outputs", + skip_if_exists=False, +) + +############PLOTTING############# +plot_full_schedule = True +draw_dependencies = True +plot_data_transfer = True +section_start_percent = (0,) +percent_shown = (100,) +################################# + +#########################PLOTTING PATHS############################## +timeline_fig_path_plotly = f"outputs/{experiment_id}/schedule.html" +memory_fig_path = f"outputs/{experiment_id}/memory.png" +##################################################################### + +#####################CostModelEvaluationLUT LOAD############################# +cost_lut_path = f"outputs/{experiment_id}/cost_lut_post_co.pickle" +cost_lut = CostModelEvaluationLUT(cost_lut_path) +############################################################################# + +# Plotting schedule timeline of best SCME +visualize_timeline_plotly( + scme, + draw_dependencies=draw_dependencies, + draw_communication=plot_data_transfer, + fig_path=timeline_fig_path_plotly, + cost_lut=cost_lut, +) +# Plotting memory usage of best SCME +plot_memory_usage(scme, section_start_percent, percent_shown, fig_path=memory_fig_path) diff --git a/outputs/custom_ssm.onnx b/outputs/custom_ssm.onnx new file mode 100644 index 0000000000000000000000000000000000000000..669004a5e20ead7c1ce9ea8b18c2113ef3578945 GIT binary patch literal 8163 zcmbst%W@mX5leu;viX8mluRQsMXysf0ZRg~eCUyf5Iqi3nJU>-s$w33tcW$aE4++{ zMEepZuDB|dW2$n>f!9>!k`Kr+m;98|GrP0XvpWl>OjdER+uhUM)350s)3i#%ZzhA$ z(Me_AxwCtx`QX0~;hFVby+0WB>W>Hg@x<*<+Jot2IGwZ`l@%3Xn}Vda2J>&-?$oQ9 zW#f4vm&>)zjRHJ@dB2ZKA&aABg&bcF-bpD7G0e^*0w$oLvfdA=zVJHJBX8ex`;tL9 ziLybGp{fP<)El>c5P!zGQL3$i`Ns2x9ltkTN5CBH3khP6?Pf}XL}i-dpa1#fE;Gen z4anA5pd`Ool(5jRI-M9+i7bRLJCBc=1&i-I|M+Cm9{b;V>n7^YHsl^r01`!x(0&JBkBIL<0__4rX)NSO6(b+SJ908Dl*^qxWEOgrvVvp>I+cWJp7?L2 z+{iQtv-9|Xhav={7^1|2!Tz+HVt_C^j}LezOhl<*k!8r8(CF4$Xc|U9Vq}p(Lsl3A zT2fdY^!uj|9_+gl#^e63wO;r8?crz;)FW7&f~yIPT?{FHd<+~|E1~#_DkAX}|W}v9(;MAzvx5&r2rLJy4lqcpVJBnhd+sF%2&P zvrX{^FOrHjF$Z6b-2Qkt7<+OtNT9tC<1y6>LvPe;@sTUcE+uIE6+YxpfsM0{0>%PuJdm|uYq-q?1ZG(5(Uq+##7`h zjUibeXRMI*1OK=;@JV%DqF}oe;o)Tl8wBgvSXVfhR*i}&Kw&+2iV=baO&qcdF^uG1 zMdaR!X<+V|2FbN&Sb$b{-#qqzVp6UwFrq6hG}eKHwK+Z>|*WLc@U*b#u+{nY944BCqjgZ_M_AFePN2e)8Ruh<5 ziJL@;GLwy3oD8`dL$1ePSYoIi$G##%j)ODX`FiHgq)te~-6{}L`xlSY#iVwLNUr@aH}-%#J=UUzXq`lkKlRBh~*75F7Pyf0}0?8Q3KGi{FqK=KGk))&!0v#@x0=9&zWbi}E@O#xlVapZ zE)98IPw->r_g`BdP-dy~zY>mO(ar>#c#otnUTOUM?)SUQPyaCBqFumj!Kz7>DMT^p z-nsaIGLUTSD2M%4G~qX! z4Ri|1)6f0oBves3&g2-jIvWgT2{ud4_>V&%p3rI81 zX&8=P;Sk)~;YX0VkSPNWUKX&mYxp?-{O~8@eB>A<*PE(p=3&;M!Z3G{Q z6{3EU3ouKlP!%dD1xR>I;a9@~z_nE>(*G-4=2lv^a_^TRJF*n>S)BKJ!{5SFnd_+* zn@qcA-i7H_oPQqNP{%*V;n7v%(Vq`KNtY*`nps;kJR{V>Q>XE!>Nzs}OH&mvvUKAzTt|H^JPo4rjvWnnvKYaO*|WEG2X5iYEikR( zwEw2vZp3fR_|1vmwu4tQM&^64hrA^9PP}e6e%p)R?#6HTg5O>=$67y!2gqBlz7Gpl ziEG?!$k(QPb>!=Iut8IT%)#tbDdNHQGCk~GQsRs3sS_>Kz`fj(3hv0)UHQ5vU+)IZ ztXXwE)Adf)6li6yQECU75M>1B16F3!6JJKUHq@~}$rZI0hLvI~*t>8h(j||9=hFqF zl4v37s!A6(5~7vS&MKOOH`04Zj~Uu^PP)e@1zc}0%^FhB^=q0lx^=B1*B;H4%l`v% CE+ None: # type: ignore """Protect the original shape attribute to prevent errors""" - raise ValueError("The numpy shape of NodeTensor is hidden in an abstraction layer") + raise ValueError("The numpy shape of NodeTensor is hidden in an abstraction layer. Call `tensor_shape` instead") @property def full_shape(self): diff --git a/stream/onnx_utils.py b/stream/onnx_utils.py new file mode 100644 index 00000000..07dba98d --- /dev/null +++ b/stream/onnx_utils.py @@ -0,0 +1,26 @@ +from onnx import ModelProto, NodeProto +from zigzag.parser.onnx.utils import get_onnx_tensor_type + + +def get_onnx_input_shapes(node: NodeProto, onnx_model: ModelProto) -> list[list[int]]: + """Return the shape of each input operand""" + input_names = node.input + input_shapes = [get_onnx_tensor_type(name, onnx_model).shape for name in input_names] + return input_shapes + + +def get_onnx_output_shapes(node: NodeProto, onnx_model: ModelProto) -> list[list[int]]: + """Return the shape of each output operand""" + + output_names = node.output + output_shapes = [get_onnx_tensor_type(name, onnx_model).shape for name in output_names] + return output_shapes + + +def has_asymmetric_input_data(node: NodeProto, onnx_model: ModelProto): + """Return true iff the node has two inputs and the input nodes have a different shape""" + if len(node.input) != 2: + return False + + input_shape1, input_shape2 = get_onnx_input_shapes(node, onnx_model) + return input_shape1 != input_shape2 diff --git a/stream/parser/onnx/asymmetric_simd.py b/stream/parser/onnx/asymmetric_simd.py index a5ca54e7..027d0af3 100644 --- a/stream/parser/onnx/asymmetric_simd.py +++ b/stream/parser/onnx/asymmetric_simd.py @@ -1,12 +1,9 @@ from typing import Any -from zigzag.parser.onnx.utils import ( - get_node_input_output_dimension_shapes, -) from zigzag.parser.workload_factory import LayerNodeFactory +from stream.onnx_utils import get_onnx_input_shapes, get_onnx_output_shapes from stream.parser.onnx.operator_parser import OnnxComputeOperatorParser -from stream.utils import get_onnx_input_shapes from stream.workload.computation.computation_node import ComputationNode @@ -30,7 +27,7 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ data["name"] = self.node.name data["operator_type"] = self.node.op_type data["operand_source"] = self.get_operand_source_input_format() - data["operand_precision"] = self.get_operand_precision_input_format() + data["operand_precision"] = self.get_operand_precision_user_format() data["dimension_relations"] = [] data["loop_sizes"] = output_shape @@ -41,8 +38,15 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ def generate_node(self): # Get the input and output activation shapes - input_shape1, input_shape2 = get_onnx_input_shapes(self.node, self.onnx_model) - _, output_shape = get_node_input_output_dimension_shapes(self.node, self.onnx_model) + input_shapes = get_onnx_input_shapes(self.node, self.onnx_model) + if len(input_shapes) != 2: + raise NotImplementedError("Only SIMD nodes with input length 2 are supported") + input_shape1, input_shape2 = input_shapes + + output_shapes = get_onnx_output_shapes(self.node, self.onnx_model) + if len(output_shapes) != 1: + raise NotImplementedError("Only SIMD nodes with input length 2 are supported") + output_shape = output_shapes.pop() if input_shape1 == output_shape: non_batched_input_shape = input_shape2 diff --git a/stream/parser/onnx/conv.py b/stream/parser/onnx/conv.py index 939a8556..a5bc8aff 100644 --- a/stream/parser/onnx/conv.py +++ b/stream/parser/onnx/conv.py @@ -19,44 +19,59 @@ class ConvParser(OnnxComputeOperatorParser): OP_TYPE = "conv" - def get_layer_node_user_format( # type: ignore + def get_layer_node_user_format( self, - kernel_shape: list[int], - strides: list[int], - dilations: list[int], - group_size: int, - padding: list[int], - ia_shape: list[int], - oa_shape: list[int], + input_shape: list[int], + output_shape: list[int], ) -> dict[str, Any]: """ Generate the necessary dictionary items required for the LayerNode creation. """ - # convert the data types to precisions based on the onnx definition + predecessors = self.get_node_predecessors() + + # Extract extra attributes + attrs = self.node.attribute + kernel_shape: list[int] = get_attribute_ints_with_name("kernel_shape", attrs, default=None) # type:ignore + strides: list[int] = get_attribute_ints_with_name("strides", attrs, default=[1, 1]) # type:ignore + dilations: list[int] = get_attribute_ints_with_name("dilations", attrs, default=[1, 1]) # type:ignore + group_size: int = get_attribute_ints_with_name("group", attrs, default=1) # type:ignore + padding: list[int] = get_attribute_ints_with_name("pads", attrs, default=[0, 0, 0, 0]) # type:ignore + + # 1D Conv case: append dimensions of size 1 so equation holds. Conv in FY dimension + print(kernel_shape) + if len(kernel_shape) == 1: + kernel_shape.insert(0, 1) + input_shape.append(1) + output_shape.append(1) + strides.append(1) + dilations.append(1) + assert len(input_shape) == 4 + assert len(output_shape) == 4 + + if len(padding) == 2: + padding = 2 * padding - # Equation data: dict[str, Any] = {} data["id"] = self.node_id - data["name"] = f"Layer{self.node_id}" + data["name"] = self.node.name data["operator_type"] = ConvParser.OP_TYPE + # IMPORTANT: If any of the input loops require padding, they should be defined as the rightmost dimensions in # the equation. This is because we construct the dimensionality order and then add the padding to those last # dimensions in the order - if group_size > 1: - data["equation"] = "O[b][g][k][oy][ox]+=W[g][c][fy][fx]*I[b][g][c][iy][ix]" - else: - data["equation"] = "O[b][g][k][oy][ox]+=W[k][c][fy][fx]*I[b][g][c][iy][ix]" + weight_dim = "g" if group_size > 1 else "k" + data["equation"] = f"O[b][g][k][oy][ox]+=W[{weight_dim}][c][fy][fx]*I[b][g][c][iy][ix]" # Get dimension sizes from input parameters - assert ia_shape[0] == oa_shape[0], "Batch size is different for input and output activations." - B = oa_shape[0] + assert input_shape[0] == output_shape[0], "Batch size is different for input and output activations." + B = output_shape[0] G = group_size - K = ceil(oa_shape[1] / G) - OX = oa_shape[3] - OY = oa_shape[2] - C = ceil(ia_shape[1] / G) - IX = ia_shape[3] - IY = ia_shape[2] + K = ceil(output_shape[1] / G) + OX = output_shape[3] + OY = output_shape[2] + C = ceil(input_shape[1] / G) + IX = input_shape[3] + IY = input_shape[2] FX = kernel_shape[0] FY = kernel_shape[1] data["loop_dims"] = ["B", "K", "G", "OX", "OY", "C", "FX", "FY"] @@ -68,7 +83,8 @@ def get_layer_node_user_format( # type: ignore f"ix={strides[0]}*ox+{dilations[0]}*fx", f"iy={strides[1]}*oy+{dilations[1]}*fy", ] - data["operand_precision"] = {"O": 16, "O_final": 8, "W": 8, "I": 8} + data["operand_precision"] = self.get_operand_precision_user_format() + data["operand_source"] = self.get_operand_source_user_format(predecessors) # Add information wrt how this conv node's input/output tensors # are represented in the onnx model vs how they are represented in the equation above. @@ -83,49 +99,16 @@ def get_layer_node_user_format( # type: ignore [padding[1], padding[3]], ] - # Find the previous layer(s) that should be this node's parent(s) - node_inputs = self.node.input - assert len(node_inputs) >= 2, f"Conv should have at least two input names, but has: {node_inputs}." - (first_input_name, second_input_name) = node_inputs[:2] - - source_list_I = [ - src for (src, src_output_names) in self.nodes_outputs.items() if first_input_name in src_output_names - ] - source_list_W = [ - src for (src, src_output_names) in self.nodes_outputs.items() if second_input_name in src_output_names - ] - assert len(source_list_I) <= 1 - assert len(source_list_W) <= 1 - - source_I = source_list_I[0] if len(source_list_I) == 1 else self.node_id - source_W = source_list_W[0] if len(source_list_W) == 1 else self.node_id - - data["operand_source"] = { - "I": source_I, - "W": source_W, - } - return data def generate_node(self): - attrs = self.node.attribute - kernel_shape: list[int] = get_attribute_ints_with_name("kernel_shape", attrs, default=None) # type:ignore - strides: list[int] = get_attribute_ints_with_name("strides", attrs, default=[1, 1]) # type:ignore - dilations: list[int] = get_attribute_ints_with_name("dilations", attrs, default=[1, 1]) # type:ignore - group_size: int = get_attribute_ints_with_name("group", attrs, default=1) # type:ignore - padding: list[int] = get_attribute_ints_with_name("pads", attrs, default=[0, 0, 0, 0]) # type:ignore # Get the input and output activation shapes - ia_dimension_shape, oa_dimension_shape = get_node_input_output_dimension_shapes(self.node, self.onnx_model) + input_shape, output_shape = get_node_input_output_dimension_shapes(self.node, self.onnx_model) node_data: dict[str, Any] = self.get_layer_node_user_format( - kernel_shape, - strides, - dilations, - group_size, - padding, - ia_dimension_shape, - oa_dimension_shape, + input_shape, + output_shape, ) node_factory = LayerNodeFactory(node_data, mapping_data=None) diff --git a/stream/parser/onnx/einsum.py b/stream/parser/onnx/einsum.py new file mode 100644 index 00000000..003ed5ab --- /dev/null +++ b/stream/parser/onnx/einsum.py @@ -0,0 +1,92 @@ +import logging +import re +from typing import Any + +from stream.onnx_utils import get_onnx_input_shapes, get_onnx_output_shapes +from stream.parser.onnx.operator_parser import OnnxComputeOperatorParser + +logger = logging.getLogger(__name__) + + +class EinsumParser(OnnxComputeOperatorParser): + + def get_einsum_equation(self): + ATTR_NAME = "equation" + + attrs_names = [attr.name for attr in self.node.attribute] + name_idx = attrs_names.index(ATTR_NAME) + value = self.node.attribute[name_idx] + return str(value) + + def get_layer_dims_per_op(self): + einsum_equation = self.get_einsum_equation() + + return re.split(",|->", einsum_equation) + + def get_layer_equation(self, layer_dims_per_op: list[str]): + def put_in_brackets(s: str): + """e.g. `abc` -> `[a][b][c]""" + return "".join([f"[{char}]" for char in s]) + + if len(layer_dims_per_op) != 3: + raise NotImplementedError + + dims_I, dims_W, dims_O = layer_dims_per_op + equation = f"O{put_in_brackets(dims_O)}+=I{put_in_brackets(dims_I)}*{put_in_brackets(dims_W)}" + return equation + + # def get_layer_dims(self, layer_dims_per_op: list[str]): + # all_dims = {char.upper() for group in layer_dims_per_op for char in group} + # return list(all_dims) + + def get_layer_dim_sizes_dict(self, layer_dims_per_op: list[str]): + input_shapes = get_onnx_input_shapes(self.node, self.onnx_model) + output_shapes = get_onnx_output_shapes(self.node, self.onnx_model) + + if len(output_shapes) != 1: + raise ValueError("Einsum with more than one output not supported") + + shapes = input_shapes + output_shapes + + if len(layer_dims_per_op) != len(shapes): + raise ValueError("Einsum equation has more parts than node inputs") + + sizes_dict: dict[str, int] = {} + for layer_dims, sizes in zip(layer_dims_per_op, shapes): + if len(layer_dims) != len(sizes): + # TODO is the order of the equation guaranteed to be the same as the input order? + raise ValueError(f"Einsum equation part {layer_dims} and operand input shape {sizes} do not match") + for layer_dim, size in zip(layer_dims.upper(), sizes): + if layer_dim not in sizes_dict: + sizes_dict[layer_dim] = size + else: + if sizes_dict[layer_dim] != size: + raise ValueError(f"Not clear what the size of {layer_dim} is in Einsum") + + return sizes_dict + + def get_layer_node_user_format( + self, + input_shape: list[int], # Argument required because of a caller function in superclass + output_shape: list[int], # TODO put shape logic in this method for all `OnnxComputeOperatorParser` subclasses + ) -> dict[str, Any]: + """! Generate layer data in user input format for Einsum.""" + predecessors = self.get_node_predecessors() + + data: dict[str, Any] = {} + data["id"] = self.node_id + data["name"] = self.node.name + data["operator_type"] = self.node.op_type + data["dimension_relations"] = [] + data["operand_source"] = self.get_operand_source_user_format(predecessors) + data["operand_precision"] = self.get_operand_precision_user_format() + + # + layer_dims_per_op = self.get_layer_dims_per_op() + sizes_dict = self.get_layer_dim_sizes_dict(layer_dims_per_op) + + data["loop_dims"] = list(sizes_dict.keys()) + data["loop_sizes"] = list(sizes_dict.values()) + data["equation"] = self.get_layer_equation(layer_dims_per_op) + + return data diff --git a/stream/parser/onnx/model.py b/stream/parser/onnx/model.py index 3de76808..b465980c 100644 --- a/stream/parser/onnx/model.py +++ b/stream/parser/onnx/model.py @@ -5,6 +5,7 @@ from zigzag.parser.onnx.utils import parse_onnx_model_from_path from stream.hardware.architecture.accelerator import Accelerator +from stream.onnx_utils import get_onnx_input_shapes, has_asymmetric_input_data from stream.parser.onnx.asymmetric_simd import AsymmetricSimdParser from stream.parser.onnx.concat import ConcatParser from stream.parser.onnx.conv import ConvParser @@ -20,7 +21,6 @@ from stream.parser.onnx.simd import SimdParser from stream.parser.onnx.softmax import SoftmaxParser from stream.parser.onnx.transpose import TransposeParser -from stream.utils import get_onnx_input_shapes, has_asymmetric_input_data from stream.workload.mapping import InterCoreMappingAttributes from stream.workload.onnx_workload import ONNXWorkload diff --git a/stream/parser/onnx/operator_parser.py b/stream/parser/onnx/operator_parser.py index 343b2665..fdd7300e 100644 --- a/stream/parser/onnx/operator_parser.py +++ b/stream/parser/onnx/operator_parser.py @@ -58,10 +58,10 @@ def run(self) -> Generator[ComputationNode, None, None]: @abstractmethod def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[int]) -> dict[str, Any]: ... - def get_operand_precision_input_format(self) -> dict[str, int]: - act_precision = self.get_activation_precision() - weight_precision = self.get_weight_precision() - intermediate_output_precision = self.get_intermediate_output_precision() + def get_operand_precision_user_format(self) -> dict[str, int]: + act_precision: int = self.get_activation_precision() + weight_precision: int = self.get_weight_precision() + intermediate_output_precision: int = self.get_intermediate_output_precision() predecessors = self.get_node_predecessors() match len(predecessors): case 1: diff --git a/stream/parser/onnx/reduce_1d.py b/stream/parser/onnx/reduce_1d.py index be4ecc1e..b34289b0 100644 --- a/stream/parser/onnx/reduce_1d.py +++ b/stream/parser/onnx/reduce_1d.py @@ -20,7 +20,7 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ data["name"] = self.node.name data["operator_type"] = self.node.op_type data["operand_source"] = self.get_operand_source_input_format() - data["operand_precision"] = self.get_operand_precision_input_format() + data["operand_precision"] = self.get_operand_precision_user_format() data["dimension_relations"] = [] data["loop_sizes"] = input_shape diff --git a/stream/parser/onnx/simd.py b/stream/parser/onnx/simd.py index 2c5ae21d..9dae37d3 100644 --- a/stream/parser/onnx/simd.py +++ b/stream/parser/onnx/simd.py @@ -22,7 +22,7 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ data["name"] = self.node.name data["operator_type"] = self.node.op_type data["operand_source"] = self.get_operand_source_input_format() - data["operand_precision"] = self.get_operand_precision_input_format() + data["operand_precision"] = self.get_operand_precision_user_format() data["dimension_relations"] = [] data["loop_sizes"] = output_shape diff --git a/stream/parser/onnx/softmax.py b/stream/parser/onnx/softmax.py index 3f0a0506..25703b26 100644 --- a/stream/parser/onnx/softmax.py +++ b/stream/parser/onnx/softmax.py @@ -93,7 +93,7 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ data["name"] = self.node.name data["operator_type"] = self.node.op_type data["operand_source"] = self.get_operand_source_input_format() - data["operand_precision"] = self.get_operand_precision_input_format() + data["operand_precision"] = self.get_operand_precision_user_format() data["dimension_relations"] = [] data["loop_sizes"] = input_shape diff --git a/stream/utils.py b/stream/utils.py index 67f93f7a..06328b57 100644 --- a/stream/utils.py +++ b/stream/utils.py @@ -4,11 +4,9 @@ from typing import TYPE_CHECKING, Any, TypeAlias from numpy.typing import NDArray -from onnx import ModelProto, NodeProto from zigzag.cost_model.cost_model import CostModelEvaluation from zigzag.datatypes import MemoryOperand from zigzag.mapping.data_movement import FourWayDataMoving -from zigzag.parser.onnx.utils import get_onnx_tensor_type from stream.hardware.architecture.core import Core from stream.workload.mapping import TILING_T @@ -21,25 +19,6 @@ ARRAY_T: TypeAlias = NDArray[Any] -def get_onnx_input_shapes(node: NodeProto, onnx_model: ModelProto) -> tuple[list[int], list[int]]: - if len(node.input) != 2: - raise ValueError(f"Node {node.name} does not have two inputs") - input_name1 = node.input[0] - input_name2 = node.input[1] - input_shape1 = get_onnx_tensor_type(input_name1, onnx_model).shape - input_shape2 = get_onnx_tensor_type(input_name2, onnx_model).shape - return input_shape1, input_shape2 - - -def has_asymmetric_input_data(node: NodeProto, onnx_model: ModelProto): - """Return true iff the node has two inputs and the input nodes have a different shape""" - if len(node.input) != 2: - return False - - input_shape1, input_shape2 = get_onnx_input_shapes(node, onnx_model) - return input_shape1 != input_shape2 - - def get_too_large_operands(cme: CostModelEvaluation, accelerator: "Accelerator", core_id: int) -> list[MemoryOperand]: """Create a list of memory operands for which an extra memory level (i.e. offchip) was added. From 742a5f917c34d587d7763f2b5689ce94dd3cf55f Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Tue, 5 Nov 2024 16:57:43 +0100 Subject: [PATCH 2/8] add SplitNode --- stream/node_tensor.py | 4 ++ stream/onnx_utils.py | 65 ++++++++++++++++++- stream/parser/onnx/asymmetric_simd.py | 2 + stream/parser/onnx/concat.py | 23 ++++--- stream/parser/onnx/conv.py | 2 + stream/parser/onnx/default.py | 2 + stream/parser/onnx/elementwise.py | 3 + stream/parser/onnx/flatten.py | 9 ++- stream/parser/onnx/gather.py | 14 +--- stream/parser/onnx/lpnormalization.py | 3 + stream/parser/onnx/model.py | 3 + stream/parser/onnx/operator_parser.py | 7 +- stream/parser/onnx/pooling.py | 2 + stream/parser/onnx/reshape.py | 2 + stream/parser/onnx/split.py | 32 +++++++++ stream/parser/onnx/transpose.py | 2 + .../generation/tiled_workload_generation.py | 49 ++++---------- .../workload/computation/computation_node.py | 2 + .../dependency_propagation/concat_node.py | 20 ++---- .../dependency_propagation/dummy_node.py | 19 +++--- .../elementwise_node.py | 19 +++--- .../dependency_propagation/flatten_node.py | 31 ++++----- .../dependency_propagation/gather_node.py | 20 ++---- .../propagation_node.py | 28 ++++++++ .../dependency_propagation/reshape_node.py | 22 ++----- .../dependency_propagation/split_node.py | 56 ++++++++++++++++ .../dependency_propagation/transpose_node.py | 28 +++----- stream/workload/node.py | 15 +++-- 28 files changed, 309 insertions(+), 175 deletions(-) create mode 100644 stream/parser/onnx/split.py create mode 100644 stream/workload/dependency_propagation/propagation_node.py create mode 100644 stream/workload/dependency_propagation/split_node.py diff --git a/stream/node_tensor.py b/stream/node_tensor.py index 98aed143..ad7fdb62 100644 --- a/stream/node_tensor.py +++ b/stream/node_tensor.py @@ -125,6 +125,10 @@ def gather(self, gather_indices: int | list[int], axis: int) -> "NodeTensor": axis = axis - 1 if axis < 0 else axis return (np.take(self.as_ndarray(), gather_indices, axis=axis)).view(NodeTensor) + def split(self, split_indices: list[int], axis: int) -> "list[NodeTensor]": + axis = axis - 1 if axis < 0 else axis + return [t.view(NodeTensor) for t in np.split(self.as_ndarray(), split_indices, axis=axis)] + def concat_with_empty(self, shape: tuple[int, ...], axis: int, variable_input_first: bool): empty_shape = self.convert_to_full_shape(shape) empty_tensor = np.zeros(empty_shape, dtype=object) diff --git a/stream/onnx_utils.py b/stream/onnx_utils.py index 07dba98d..150d3701 100644 --- a/stream/onnx_utils.py +++ b/stream/onnx_utils.py @@ -1,6 +1,40 @@ -from onnx import ModelProto, NodeProto +from onnx import AttributeProto, ModelProto, NodeProto, numpy_helper from zigzag.parser.onnx.utils import get_onnx_tensor_type +import numpy as np +import onnx + + +def get_attribute_as_ints( + node: NodeProto, attribute_name: str, default: list[int] | int | None = None +) -> list[int] | int: + """! Return the value of an attribute of given name from the given attributes + If name does not exist in attrs, the default provided by the caller is used. + If the caller doesn't supply a default, an error is thrown. + + """ + attrs = node.attribute + attrs_names = [attr.name for attr in attrs] + try: + name_idx = attrs_names.index(attribute_name) + value = attrs[name_idx] + attr_type = value.type + if attr_type == AttributeProto.AttributeType.INT: # type: ignore + return int(value.i) + elif attr_type == AttributeProto.AttributeType.INTS: # type: ignore + return list(value.ints) + elif attr_type == AttributeProto.AttributeType.TENSOR: # type: ignore + return list(numpy_helper.to_array(value.t).tolist()) # type: ignore + else: + raise NotImplementedError(f"Attribute extraction of type {attr_type} not supported.") + except ValueError as exc: + if default is not None: + return default + else: + raise ValueError( + f"Node {node.name} has no attribute called {attribute_name} and no default was given. Attributes = {attrs_names}." + ) from exc + def get_onnx_input_shapes(node: NodeProto, onnx_model: ModelProto) -> list[list[int]]: """Return the shape of each input operand""" @@ -24,3 +58,32 @@ def has_asymmetric_input_data(node: NodeProto, onnx_model: ModelProto): input_shape1, input_shape2 = get_onnx_input_shapes(node, onnx_model) return input_shape1 != input_shape2 + + +def get_axis_attribute(node: NodeProto): + """Find the value of the axis associated with this ONNX node""" + ATTR_NAME = "axis" + + value = get_attribute_as_ints(node, ATTR_NAME) + if not isinstance(value, int): + raise ValueError(f"{ATTR_NAME} attribute as list of ints not supported") + return value + + +def get_split_attribute(node: NodeProto, onnx_model: ModelProto): + # ATTR_NAME = "split" + + output_name = next(n for n in node.input if "split" in n.lower()) + + for node in onnx_model.graph.node: + if node.op_type == "Constant" and node.output[0] == output_name: + for attr in node.attribute: + if attr.name == "value": + tensor = attr.t # This is an ONNX TensorProto + # Decode tensor to a numpy array + array = np.frombuffer(tensor.raw_data, dtype=int) + array = array.reshape([dim for dim in tensor.dims]) + + return [int(i) for i in array] + + raise ValueError diff --git a/stream/parser/onnx/asymmetric_simd.py b/stream/parser/onnx/asymmetric_simd.py index 027d0af3..3bedfa15 100644 --- a/stream/parser/onnx/asymmetric_simd.py +++ b/stream/parser/onnx/asymmetric_simd.py @@ -61,6 +61,7 @@ def generate_node(self): node_factory = LayerNodeFactory(node_data, mapping_data=None) node_attrs = node_factory.create_node_attr() mapping = self.get_mapping_this_node() + input_names = list(self.node.input) return ComputationNode( node_id=self.node_id, @@ -68,4 +69,5 @@ def generate_node(self): node_attr=node_attrs, mapping_attr=mapping, op_type=self.node.op_type, + input_names=input_names, ) diff --git a/stream/parser/onnx/concat.py b/stream/parser/onnx/concat.py index 3c63643e..229db1b9 100644 --- a/stream/parser/onnx/concat.py +++ b/stream/parser/onnx/concat.py @@ -7,10 +7,21 @@ class ConcatParser(OnnxOperatorParser): """Parses an onnx gather operator into a ConcatNode.""" + def get_axis_value(self): + AXIS_ATTR = "axis" + + """Find the value of the axis associated with this concat node in ONNX""" + # `axis` is an attribute of the node + try: + axis_attr = next(filter(lambda x: x.name == AXIS_ATTR, self.node.attribute)) + return axis_attr.i + except StopIteration: + raise ValueError("Axis attribute not found in ONNX node") + def generate_node(self): predecessors = self.get_node_predecessors() - axis = self.get_axis_value() + input_names = list(self.node.input) input_1, input_2 = self.node.input[0], self.node.input[1] @@ -36,13 +47,5 @@ def generate_node(self): axis=axis, constant_shape=constant_shape, variable_input_first=variable_input_first, + input_names=input_names, ) - - def get_axis_value(self): - """Find the value of the axis associated with this concat node in ONNX""" - # `axis` is an attribute of the node - try: - axis_attr = next(filter(lambda x: x.name == "axis", self.node.attribute)) - return axis_attr.i - except StopIteration: - raise ValueError("Axis attribute not found in ONNX node") diff --git a/stream/parser/onnx/conv.py b/stream/parser/onnx/conv.py index a5bc8aff..10f8566e 100644 --- a/stream/parser/onnx/conv.py +++ b/stream/parser/onnx/conv.py @@ -114,6 +114,7 @@ def generate_node(self): node_factory = LayerNodeFactory(node_data, mapping_data=None) node_attrs = node_factory.create_node_attr() mapping = self.get_mapping_this_node() + input_names = list(self.node.input) return ComputationNode( node_id=self.node_id, @@ -122,4 +123,5 @@ def generate_node(self): mapping_attr=mapping, op_type=ConvParser.OP_TYPE, operand_tensor_reshape=None, + input_names=input_names, ) diff --git a/stream/parser/onnx/default.py b/stream/parser/onnx/default.py index 8bdd3f99..645fc88a 100644 --- a/stream/parser/onnx/default.py +++ b/stream/parser/onnx/default.py @@ -7,10 +7,12 @@ class DefaultNodeParser(OnnxOperatorParser): def generate_node(self): predecessors = self.get_node_predecessors() + input_names = list(self.node.input) return DummyNode( node_id=self.node_id, node_name=self.node.name, predecessors=predecessors, op_type=self.node.op_type.lower(), + input_names=input_names, ) diff --git a/stream/parser/onnx/elementwise.py b/stream/parser/onnx/elementwise.py index d7b68a55..55e035d8 100644 --- a/stream/parser/onnx/elementwise.py +++ b/stream/parser/onnx/elementwise.py @@ -14,6 +14,8 @@ def __init__(self, node_id, node, nodes_outputs, mapping, onnx_model) -> None: self.name = node.name def generate_node(self): + input_names = list(self.node.input) + # Get the predecessors of this node predecessors = [] for node_input in self.node.input: @@ -28,5 +30,6 @@ def generate_node(self): node_id=self.node_id, node_name=self.name, predecessor=predecessors, + input_names=input_names, ) return node_obj diff --git a/stream/parser/onnx/flatten.py b/stream/parser/onnx/flatten.py index 215f6676..35e4f0c0 100644 --- a/stream/parser/onnx/flatten.py +++ b/stream/parser/onnx/flatten.py @@ -1,5 +1,3 @@ -from zigzag.parser.onnx.utils import get_attribute_ints_with_name - from stream.parser.onnx.operator_parser import OnnxOperatorParser from stream.workload.dependency_propagation.flatten_node import FlattenNode @@ -12,12 +10,13 @@ def generate_node(self): assert len(predecessors) == 1 predecessor = predecessors[0] - attrs = self.node.attribute - # Get the axis which indicates how to flatten the input tensor - axis: int | None = get_attribute_ints_with_name("axis", attrs, default=None) # type: ignore + input_names = list(self.node.input) + axis = self.get_axis_attribute() + return FlattenNode( node_id=self.node_id, node_name=self.node.name, predecessor=predecessor, axis=axis, + input_names=input_names, ) diff --git a/stream/parser/onnx/gather.py b/stream/parser/onnx/gather.py index e7a32cc9..b9c3fde2 100644 --- a/stream/parser/onnx/gather.py +++ b/stream/parser/onnx/gather.py @@ -9,8 +9,9 @@ class GatherParser(OnnxOperatorParser): def generate_node(self): predecessors = self.get_node_predecessors() - axis = self.get_axis_value() + axis = self.get_axis_attribute() indices = self.get_indices_value() + input_names = list(self.node.input) return GatherNode( node_id=self.node_id, @@ -18,6 +19,7 @@ def generate_node(self): predecessors=predecessors, gather_axis=axis, gather_indices=indices, + input_names=input_names, ) def get_indices_value(self): @@ -39,13 +41,3 @@ def get_indices_value(self): indices = DEFAULT return indices - - def get_axis_value(self): - """Find the value of the axis associated with this gather node in ONNX""" - # `axis` is an attribute of the node - try: - axis_attr = next(filter(lambda x: x.name == "axis", self.node.attribute)) - axis = axis_attr.i - except StopIteration: - axis = 0 - return axis diff --git a/stream/parser/onnx/lpnormalization.py b/stream/parser/onnx/lpnormalization.py index 0ca2569f..6f4ddc5b 100644 --- a/stream/parser/onnx/lpnormalization.py +++ b/stream/parser/onnx/lpnormalization.py @@ -11,6 +11,8 @@ def __init__(self, node_id, node, nodes_outputs, mapping, onnx_model) -> None: super().__init__(node_id, node, nodes_outputs, mapping, onnx_model) def generate_node(self): + input_names = list(self.node.input) + # Get the predecessors of this node # TODO use superclass' `get_node_predecessors` predecessors = [] @@ -23,5 +25,6 @@ def generate_node(self): node_id=self.node_id, node_name=self.node_name, predecessor=self.predecessor, + input_names=input_names, ) return node_obj diff --git a/stream/parser/onnx/model.py b/stream/parser/onnx/model.py index b465980c..a648fba8 100644 --- a/stream/parser/onnx/model.py +++ b/stream/parser/onnx/model.py @@ -20,6 +20,7 @@ from stream.parser.onnx.reshape import ReshapeParser from stream.parser.onnx.simd import SimdParser from stream.parser.onnx.softmax import SoftmaxParser +from stream.parser.onnx.split import SplitParser from stream.parser.onnx.transpose import TransposeParser from stream.workload.mapping import InterCoreMappingAttributes from stream.workload.onnx_workload import ONNXWorkload @@ -46,12 +47,14 @@ class ONNXModelParser: "Relu": SimdParser, "Gelu": SimdParser, "Silu": SimdParser, + # Dependency propagation "LpNormalization": LpNormalizationParser, "Gather": GatherParser, "Transpose": TransposeParser, "Reshape": ReshapeParser, "Flatten": FlattenParser, "Concat": ConcatParser, + "Split": SplitParser, } def __init__( diff --git a/stream/parser/onnx/operator_parser.py b/stream/parser/onnx/operator_parser.py index fdd7300e..a4345895 100644 --- a/stream/parser/onnx/operator_parser.py +++ b/stream/parser/onnx/operator_parser.py @@ -7,6 +7,7 @@ from zigzag.parser.workload_factory import LayerNodeFactory from stream.hardware.architecture.accelerator import Accelerator +from stream.onnx_utils import get_axis_attribute from stream.workload.computation.computation_node import ComputationNode from stream.workload.mapping import InterCoreMappingAttributes from stream.workload.node import Node @@ -49,6 +50,9 @@ def get_operand_source_input_format(self): case _: raise ValueError("No more than 2 layer predecessors expected") + def get_axis_attribute(self): + return get_axis_attribute(self.node) + class OnnxComputeOperatorParser(OnnxOperatorParser, metaclass=ABCMeta): @@ -120,8 +124,8 @@ def generate_node(self): node_data = self.get_layer_node_user_format(input_shape, output_shape) node_factory = LayerNodeFactory(node_data, mapping_data=[]) node_attrs = node_factory.create_node_attr() - mapping = self.get_mapping_this_node() + input_names = list(self.node.input) return ComputationNode( node_id=self.node_id, @@ -129,4 +133,5 @@ def generate_node(self): op_type=self.node.op_type, node_attr=node_attrs, mapping_attr=mapping, + input_names=input_names, ) diff --git a/stream/parser/onnx/pooling.py b/stream/parser/onnx/pooling.py index ff120f9f..780efbec 100644 --- a/stream/parser/onnx/pooling.py +++ b/stream/parser/onnx/pooling.py @@ -117,10 +117,12 @@ def generate_node(self): node_factory = LayerNodeFactory(node_data, None) node_attrs = node_factory.create_node_attr() mapping = self.get_mapping_this_node() + input_names = list(self.node.input) return PoolingNode( node_id=self.node_id, node_name=self.node.name, node_attr=node_attrs, mapping_attr=mapping, + input_names=input_names, ) diff --git a/stream/parser/onnx/reshape.py b/stream/parser/onnx/reshape.py index 325eb378..1ed9c193 100644 --- a/stream/parser/onnx/reshape.py +++ b/stream/parser/onnx/reshape.py @@ -14,10 +14,12 @@ def generate_node(self): # The operator shape is saved as the second input, so we need to get the input's dimension shape shape = tuple(get_node_input_output_dimension_shapes(self.node, self.onnx_model)[1]) + input_names = list(self.node.input) return ReshapeNode( node_id=self.node_id, node_name=self.node.name, predecessor=predecessor, shape=shape, + input_names=input_names, ) diff --git a/stream/parser/onnx/split.py b/stream/parser/onnx/split.py new file mode 100644 index 00000000..95d1967b --- /dev/null +++ b/stream/parser/onnx/split.py @@ -0,0 +1,32 @@ +from stream.onnx_utils import get_split_attribute +from stream.parser.onnx.operator_parser import OnnxOperatorParser +from stream.workload.dependency_propagation.split_node import SplitNode + + +class SplitParser(OnnxOperatorParser): + """Parses an onnx gather operator into a SplitNode.""" + + def generate_node(self): + # Single predecessor + predecessors = self.get_node_predecessors() + if len(predecessors) > 1: + raise ValueError("Split node should not have more than one input") + predecessor = predecessors.pop() + + axis = self.get_axis_attribute() + splits = get_split_attribute(self.node, self.onnx_model) + input_names = list(self.node.input) + output_names = list(self.node.output) + + if len(splits) != len(output_names): + raise ValueError + + return SplitNode( + node_id=self.node_id, + node_name=self.node.name, + predecessor=predecessor, + axis=axis, + splits=splits, + input_names=input_names, + output_names=output_names, + ) diff --git a/stream/parser/onnx/transpose.py b/stream/parser/onnx/transpose.py index ba6dae2a..0b3bcb7a 100644 --- a/stream/parser/onnx/transpose.py +++ b/stream/parser/onnx/transpose.py @@ -11,12 +11,14 @@ def generate_node(self): predecessor = predecessors.pop() permute_axes = self.get_permute_indices() + input_names = list(self.node.input) return TransposeNode( node_id=self.node_id, node_name=self.node.name, predecessor=predecessor, permute_axes=permute_axes, + input_names=input_names, ) def get_permute_indices(self): diff --git a/stream/stages/generation/tiled_workload_generation.py b/stream/stages/generation/tiled_workload_generation.py index df5b4953..92853964 100644 --- a/stream/stages/generation/tiled_workload_generation.py +++ b/stream/stages/generation/tiled_workload_generation.py @@ -19,12 +19,7 @@ from stream.workload.computation.computation_node import ComputationNode, LoopRanges from stream.workload.dependency_propagation.concat_node import ConcatNode from stream.workload.dependency_propagation.dummy_node import DummyNode -from stream.workload.dependency_propagation.elementwise_node import ElementwiseNode -from stream.workload.dependency_propagation.flatten_node import FlattenNode -from stream.workload.dependency_propagation.gather_node import GatherNode -from stream.workload.dependency_propagation.lpnormalization_node import LpNormalizationNode -from stream.workload.dependency_propagation.reshape_node import ReshapeNode -from stream.workload.dependency_propagation.transpose_node import TransposeNode +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.dnn_workload import DNNWorkloadStream from stream.workload.node import Node from stream.workload.onnx_workload import ComputationNodeWorkload, ONNXWorkload @@ -585,6 +580,7 @@ def get_tensor_cn_for_op(node: ComputationNode, dependent_operand: LayerOperand) assert ( len(paths_between) > 0 ), "No paths between producer and consumer found without ComputationNode in intermediates." + for path_between in paths_between: # First node in the path is a ComputationNode, of which we extract the output operand dependency tensor first_node = path_between[0] @@ -592,10 +588,10 @@ def get_tensor_cn_for_op(node: ComputationNode, dependent_operand: LayerOperand) tensor = get_tensor_cn_for_op(first_node, dependent_operand=Constants.OUTPUT_LAYER_OP) # Propagate through intermediate, non-computation nodes - for _, node in enumerate(path_between[1:-1], start=1): - if isinstance(node, ComputationNode): - raise ValueError("Intermediate nodes should not be of type ComputationNode.") - tensor = self.propagate_cn_production_for_non_cn(node, tensor) + for i, node in enumerate(path_between[1:-1], start=1): + assert isinstance(node, PropagationNode), "Intermediate nodes should not be of type ComputationNode" + next_node = path_between[i + 1] + tensor = node.propagate(tensor, next_node) # Final node: Computation node final_node: ComputationNode = path_between[-1] # type: ignore @@ -607,7 +603,7 @@ def get_tensor_cn_for_op(node: ComputationNode, dependent_operand: LayerOperand) ) # Error handling of shape mismatches in tensor propagation - def get_final_tensor_alt_operand(): + def _get_final_tensor_alt_operand(): """Error handling case 1: sources for `W` and `I` operand are swapped for this node -> try the other one""" try: @@ -617,7 +613,7 @@ def get_final_tensor_alt_operand(): raise TensorDimensionMismatchException return get_tensor_cn_for_op(final_node, alt_operand) - def get_shape_inferred_propagated_tensor(tensor: NodeTensor, final_tensor: NodeTensor): + def _get_shape_inferred_propagated_tensor(tensor: NodeTensor, final_tensor: NodeTensor): """Error handling case 2: dimensions of ComputationNode (`final_tensor`) were altered by stream (e.g. to be properly divisible) but this is not reflected in `ConcatNode` with constant shape. -> manually fix shape""" @@ -644,17 +640,17 @@ def get_shape_inferred_propagated_tensor(tensor: NodeTensor, final_tensor: NodeT inter_edges = self.get_inter_edges_tensor_based(tensor, final_tensor) except TensorDimensionMismatchException: try: # Error case 1 - final_tensor = get_final_tensor_alt_operand() + final_tensor = _get_final_tensor_alt_operand() inter_edges = self.get_inter_edges_tensor_based(tensor, final_tensor) except TensorDimensionMismatchException: try: # Error case 2 final_tensor = get_tensor_cn_for_op(final_node, dependent_operand) - tensor = get_shape_inferred_propagated_tensor(tensor, final_tensor) + tensor = _get_shape_inferred_propagated_tensor(tensor, final_tensor) inter_edges = self.get_inter_edges_tensor_based(tensor, final_tensor) except TensorDimensionMismatchException: # Error case 1 and 2 combined - final_tensor = get_final_tensor_alt_operand() - tensor = get_shape_inferred_propagated_tensor(tensor, final_tensor) + final_tensor = _get_final_tensor_alt_operand() + tensor = _get_shape_inferred_propagated_tensor(tensor, final_tensor) inter_edges = self.get_inter_edges_tensor_based(tensor, final_tensor) for producer, cons in inter_edges: @@ -670,27 +666,6 @@ def get_shape_inferred_propagated_tensor(tensor: NodeTensor, final_tensor: NodeT ) return all_inter_edges - def propagate_cn_production_for_non_cn(self, node: Node, input_tensor: NodeTensor) -> NodeTensor: - match node: - case ReshapeNode(): - return node.reshape_operand_tensor(input_tensor) - case TransposeNode(): - return node.transpose(input_tensor) - case LpNormalizationNode(): - return node.lpnormalization_operand_tensor(input_tensor) - case FlattenNode(): - return node.flatten(input_tensor) - case ElementwiseNode(): - return input_tensor.copy() - case GatherNode(): - return node.gather_operand_tensor(input_tensor) - case ConcatNode(): - return node.concat(input_tensor) - case DummyNode(): - return input_tensor - case _: - raise NotImplementedError(f"Tensor propagation not implemented for node {node.name}.") - @staticmethod def get_inter_edges_tensor_based(producer_output_tensor: NodeTensor, consumer_input_tensor: NodeTensor): """This method obtains the edges between a producer and consumer. diff --git a/stream/workload/computation/computation_node.py b/stream/workload/computation/computation_node.py index cad237c5..58e7e535 100644 --- a/stream/workload/computation/computation_node.py +++ b/stream/workload/computation/computation_node.py @@ -61,6 +61,7 @@ def __init__( produces_final_output: bool = False, group_id: int = 0, sub_id: int = -1, # To distinguish alternative versions of this node + input_names: list[str] = [], ): op_type = op_type.lower() @@ -76,6 +77,7 @@ def __init__( offchip_energy=0, runtime=0, possible_core_allocation=mapping_attr.core_allocation, + input_names=input_names, ) # Overwrite default spatial mapping with given one diff --git a/stream/workload/dependency_propagation/concat_node.py b/stream/workload/dependency_propagation/concat_node.py index 113aba48..acd956a5 100644 --- a/stream/workload/dependency_propagation/concat_node.py +++ b/stream/workload/dependency_propagation/concat_node.py @@ -1,11 +1,11 @@ from zigzag.datatypes import LayerOperand -from zigzag.workload.layer_node_abc import LayerNodeABC from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.node import Node -class ConcatNode(Node, LayerNodeABC): +class ConcatNode(PropagationNode): """Class that represents an onnx Concat node with one constant input.""" def __init__( @@ -16,6 +16,7 @@ def __init__( axis: int, constant_shape: tuple[int, ...], variable_input_first: bool, + input_names: list[str] = [], ) -> None: """Initialize the ConcatNode @@ -26,17 +27,8 @@ def __init__( variable_input_first: Wether the result is `concat(input, constant_tensor)` or `concat(constant_tensor, input)` """ - Node.__init__( - self, - node_id=node_id, - node_name=node_name, - type="gather", - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) - LayerNodeABC.__init__(self, node_id=node_id, node_name=node_name) + op_type = "concat" + super().__init__(node_id, node_name, op_type, input_names) self.axis = axis self.constant_shape = constant_shape @@ -53,7 +45,7 @@ def __init__( case _: raise ValueError("More than two inputs for ConcatNode") - def concat(self, tensor: NodeTensor) -> NodeTensor: + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: """Perform gather operation on the tensor.""" return tensor.concat_with_empty( shape=self.constant_shape, axis=self.axis, variable_input_first=self.variable_input_first diff --git a/stream/workload/dependency_propagation/dummy_node.py b/stream/workload/dependency_propagation/dummy_node.py index e24dc0bd..9e26f04e 100644 --- a/stream/workload/dependency_propagation/dummy_node.py +++ b/stream/workload/dependency_propagation/dummy_node.py @@ -1,9 +1,11 @@ from zigzag.workload.dummy_node import DummyNode as DummyNodeZigZag +from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.node import Node -class DummyNode(DummyNodeZigZag, Node): +class DummyNode(DummyNodeZigZag, PropagationNode): """DummyNode of an onnx operator that is not import for finer graph generation or for cost estimation, but plays a role because of the passing of the input and output tensors. """ @@ -14,7 +16,9 @@ def __init__( node_name: str, predecessors: list[int], op_type: str = "dummy", + input_names: list[str] = [], ) -> None: + PropagationNode.__init__(self, node_id, node_name, op_type, input_names) DummyNodeZigZag.__init__( self, node_id=node_id, @@ -22,13 +26,6 @@ def __init__( node_type=op_type, node_name=node_name, ) - Node.__init__( - self, - node_id=node_id, - node_name=node_name, - type=op_type, - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) + + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: + return tensor diff --git a/stream/workload/dependency_propagation/elementwise_node.py b/stream/workload/dependency_propagation/elementwise_node.py index fbb507b2..47d2fa66 100644 --- a/stream/workload/dependency_propagation/elementwise_node.py +++ b/stream/workload/dependency_propagation/elementwise_node.py @@ -1,25 +1,21 @@ from zigzag.datatypes import LayerOperand +from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.node import Node -class ElementwiseNode(Node): +class ElementwiseNode(PropagationNode): def __init__( self, node_id: int, node_name: str, predecessor: int, + input_names: list[str], ) -> None: - super().__init__( - node_id=node_id, - node_name=node_name, - type="elementwise", - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) + op_type = "elementwise" + super().__init__(node_id, node_name, op_type, input_names) self.input_operand_source = {LayerOperand("I"): predecessor} def join(self, tensor1, tensor2): @@ -30,3 +26,6 @@ def join(self, tensor1, tensor2): tensor2 (np.ndarray): The second input tensor """ return tensor1 | tensor2 + + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: + return tensor diff --git a/stream/workload/dependency_propagation/flatten_node.py b/stream/workload/dependency_propagation/flatten_node.py index dbe48577..cd82be9e 100644 --- a/stream/workload/dependency_propagation/flatten_node.py +++ b/stream/workload/dependency_propagation/flatten_node.py @@ -1,12 +1,12 @@ import numpy as np from zigzag.datatypes import LayerOperand -from zigzag.workload.layer_node_abc import LayerNodeABC from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.node import Node -class FlattenNode(Node, LayerNodeABC): +class FlattenNode(PropagationNode): """Class that represents an onnx Flatten node.""" def __init__( @@ -15,32 +15,23 @@ def __init__( node_name: str, predecessor: int | None, axis: int | None, + input_names: list[str], ) -> None: """Initialize the FlattenNode Args: - shape (list): The output tensor's shape. + shape: The output tensor's shape. """ - super().__init__( - node_id=node_id, - node_name=node_name, - type="flatten", - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) + op_type = "flatten" + super().__init__(node_id, node_name, op_type, input_names) + self.axis = axis if predecessor is not None: self.input_operand_source = {LayerOperand("I"): predecessor} - def flatten(self, input_tensor: NodeTensor) -> NodeTensor: - """Reshape an input tensor - - Args: - input_tensor (np.ndarray): The input tensor - """ - shape = input_tensor.tensor_shape + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: + """Reshape an input tensor""" + shape = tensor.tensor_shape # taken from https://github.com/onnx/onnx/blob/main/docs/Operators.md#examples-51 new_shape = (1, -1) if self.axis == 0 else (np.prod(shape[0 : self.axis]).astype(int), -1) - return input_tensor.reshape(new_shape) + return tensor.reshape(new_shape) diff --git a/stream/workload/dependency_propagation/gather_node.py b/stream/workload/dependency_propagation/gather_node.py index 1967ac06..6d584072 100644 --- a/stream/workload/dependency_propagation/gather_node.py +++ b/stream/workload/dependency_propagation/gather_node.py @@ -1,11 +1,11 @@ from zigzag.datatypes import LayerOperand -from zigzag.workload.layer_node_abc import LayerNodeABC from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.node import Node -class GatherNode(Node, LayerNodeABC): +class GatherNode(PropagationNode): """Class that represents an onnx Reshape node.""" def __init__( @@ -15,6 +15,7 @@ def __init__( predecessors: list[int], gather_axis: int, gather_indices: int | list[int], + input_names: list[str] = [], ) -> None: """Initialize the GatherNode @@ -23,17 +24,8 @@ def __init__( gather_axis: Which axis to gather on. gather_indices: Indices of elements to be gathered. """ - Node.__init__( - self, - node_id=node_id, - node_name=node_name, - type="gather", - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) - LayerNodeABC.__init__(self, node_id=node_id, node_name=node_name) + op_type = "gather" + super().__init__(node_id, node_name, op_type, input_names) self.gather_axis = gather_axis self.gather_indices = gather_indices @@ -48,6 +40,6 @@ def __init__( case _: raise ValueError("More than two inputs for GatherNode") - def gather_operand_tensor(self, tensor: NodeTensor) -> NodeTensor: + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: """Perform gather operation on the tensor.""" return tensor.gather(self.gather_indices, axis=self.gather_axis) diff --git a/stream/workload/dependency_propagation/propagation_node.py b/stream/workload/dependency_propagation/propagation_node.py new file mode 100644 index 00000000..401a3242 --- /dev/null +++ b/stream/workload/dependency_propagation/propagation_node.py @@ -0,0 +1,28 @@ +from abc import abstractmethod + +from zigzag.workload.layer_node_abc import LayerNodeABC + +from stream.node_tensor import NodeTensor +from stream.workload.node import Node + + +class PropagationNode(Node, LayerNodeABC): + """Stream node that does not perform computations and is not mapped on hardware, but propagates dependencies + between nodes""" + + def __init__(self, node_id: int, node_name: str, op_type: str, input_names: list[str]): + Node.__init__( + self, + node_id=node_id, + node_name=node_name, + type=op_type, + onchip_energy=0, + offchip_energy=0, + runtime=0, + possible_core_allocation=[-1], + input_names=input_names, + ) + LayerNodeABC.__init__(self, node_id=node_id, node_name=node_name) + + @abstractmethod + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: ... diff --git a/stream/workload/dependency_propagation/reshape_node.py b/stream/workload/dependency_propagation/reshape_node.py index c1223240..33ef3537 100644 --- a/stream/workload/dependency_propagation/reshape_node.py +++ b/stream/workload/dependency_propagation/reshape_node.py @@ -1,11 +1,11 @@ +from yaml import Node from zigzag.datatypes import Constants -from zigzag.workload.layer_node_abc import LayerNodeABC from stream.node_tensor import NodeTensor -from stream.workload.node import Node +from stream.workload.dependency_propagation.propagation_node import PropagationNode -class ReshapeNode(Node, LayerNodeABC): +class ReshapeNode(PropagationNode): """Class that represents an onnx Reshape node.""" def __init__( @@ -15,6 +15,7 @@ def __init__( predecessor: int, shape: tuple[int, ...], allow_zero: bool = False, + input_names: list[str] = [], ) -> None: """Initialize the ReshapeNode @@ -23,23 +24,14 @@ def __init__( shape: The output tensor's shape. allow_zero: wether the output shape can be 0 at some dimensions. Iff True, shape `[2,0,3]` becomes `[2,3]` """ - Node.__init__( - self, - node_id=node_id, - node_name=node_name, - type="reshape", - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) - LayerNodeABC.__init__(self, node_id=node_id, node_name=node_name) + op_type = "reshape" + super().__init__(node_id, node_name, op_type, input_names) self.allow_zero = allow_zero self.shape = shape self.input_operand_source = {Constants.LAYER_OP_I: predecessor} - def reshape_operand_tensor(self, tensor: NodeTensor): + def propagate(self, tensor: NodeTensor, next_node: Node) -> NodeTensor: """Reshape the tensor back to the representation needed for producer/consumer.""" new_shape = self.shape if not new_shape: diff --git a/stream/workload/dependency_propagation/split_node.py b/stream/workload/dependency_propagation/split_node.py new file mode 100644 index 00000000..631be9f0 --- /dev/null +++ b/stream/workload/dependency_propagation/split_node.py @@ -0,0 +1,56 @@ +import numpy as np +from zigzag.datatypes import Constants + +from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode +from stream.workload.node import Node + + +class SplitNode(PropagationNode): + """Class that represents an onnx Split node.""" + + def __init__( + self, + node_id: int, + node_name: str, + predecessor: int, + axis: int, + splits: list[int], + output_names: list[str], + input_names: list[str] = [], + ) -> None: + """Initialize the SplitNode + Split the tensor at axis `axis`. The sizes are given by `splits`. `len(splits)` is the number of output nodes. + + Args: + predecessors: The id of this node's parent. + axis: axis in which to split + splits: sizes of the output splits in the given axis + output_names: the node names that correspond to the splits + """ + assert len(splits) == len(output_names) + op_type = "split" + super().__init__(node_id, node_name, op_type, input_names) + + self.axis = axis + self.splits = splits + self.input_operand_source = {Constants.LAYER_OP_I: predecessor} + self.output_names = output_names + + def propagate(self, tensor: NodeTensor, next_node: Node): + """Split the tensor back to the representation needed for producer/consumer.""" + + # Numpy requires the indices where to split instead of the sizes of the resulting splits + split_indices = list(np.cumsum(self.splits)[:-1]) + output_tensors = tensor.split(split_indices, axis=self.axis) + + # Find which split part corresponds to the input of the next node + try: + index = next(i for i, output_name in enumerate(self.output_names) if output_name in next_node.input_names) + except StopIteration: + raise ValueError( + f"Cannot find this nodes' ({self.name}) outputs {self.output_names} in next nodes' inputs {next_node.input_names}" + ) + + output_tensor = output_tensors[index] + return output_tensor diff --git a/stream/workload/dependency_propagation/transpose_node.py b/stream/workload/dependency_propagation/transpose_node.py index e4fb1223..d2d2fb23 100644 --- a/stream/workload/dependency_propagation/transpose_node.py +++ b/stream/workload/dependency_propagation/transpose_node.py @@ -1,11 +1,11 @@ from zigzag.datatypes import LayerOperand -from zigzag.workload.layer_node_abc import LayerNodeABC from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode from stream.workload.node import Node -class TransposeNode(Node, LayerNodeABC): +class TransposeNode(PropagationNode): """Class that represents an onnx Transpose node.""" def __init__( @@ -14,26 +14,14 @@ def __init__( node_name: str, predecessor: int, permute_axes: list[int] | None = None, + input_names: list[str] = [], ) -> None: - Node.__init__( - self, - node_id=node_id, - node_name=node_name, - type="reshape", - onchip_energy=0, - offchip_energy=0, - runtime=0, - possible_core_allocation=[-1], - ) - LayerNodeABC.__init__(self, node_id=node_id, node_name=node_name) + op_type = "transpose" + super().__init__(node_id, node_name, op_type, input_names) self.permute_axes = permute_axes self.input_operand_source = {LayerOperand("I"): predecessor} - def transpose(self, input_tensor: NodeTensor) -> NodeTensor: - """Transpose an input tensor. - - Args: - input_tensor (np.ndarray): The input tensor - """ - return input_tensor.transpose(axes=self.permute_axes) + def propagate(self, tensor: NodeTensor, next_node: Node | None = None) -> NodeTensor: + """Transpose an input tensor.""" + return tensor.transpose(axes=self.permute_axes) diff --git a/stream/workload/node.py b/stream/workload/node.py index 06f216ac..c720ef88 100644 --- a/stream/workload/node.py +++ b/stream/workload/node.py @@ -20,17 +20,19 @@ def __init__( possible_core_allocation: list[int], core_allocation_is_fixed: bool = False, chosen_core_allocation: int | None = None, + input_names: list[str] = [], ) -> None: """Initialize the Node metaclass Args: - type (str): The type of Node. - energy (float): The energy consumption of this Node. - runtime (int): The runtime of this Node. - possible_core_allocation (int): The core id on which this Node can be mapped. - inputs: (List[str]): The names of the input tensors of this node - outputs: (List[str]): The names of the output tensors of this node. + type: The type of Node. + energy: The energy consumption of this Node. + runtime: The runtime of this Node. + possible_core_allocation: The core id on which this Node can be mapped. + inputs: The names of the input tensors of this node + outputs: The names of the output tensors of this node. chosen_core_allocation: The final core allocation of this node + input_names: Names of the ONNX input node """ super().__init__(node_id, node_name) @@ -41,6 +43,7 @@ def __init__( self.possible_core_allocation = possible_core_allocation self.core_allocation_is_fixed = core_allocation_is_fixed self.chosen_core_allocation = chosen_core_allocation + self.input_names = input_names # will be set by the scheduler self.start = None # will be set by the scheduler From 0e3f4040c79ed69da0021be33865addb8a5389aa Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Thu, 7 Nov 2024 13:42:18 +0100 Subject: [PATCH 3/8] add slice operator --- .gitignore | 1 + outputs/custom_ssm.onnx | Bin 8163 -> 8121 bytes stream/node_tensor.py | 15 +++ stream/onnx_utils.py | 48 ++++--- stream/parser/onnx/conv.py | 122 +++++++++++------- stream/parser/onnx/einsum.py | 7 +- stream/parser/onnx/model.py | 5 + stream/parser/onnx/slice.py | 33 +++++ .../generation/tiled_workload_generation.py | 3 +- stream/stages/generation/tiling_generation.py | 58 +++++++-- .../workload/computation/computation_node.py | 23 ---- .../dependency_propagation/slice_node.py | 45 +++++++ 12 files changed, 258 insertions(+), 102 deletions(-) create mode 100644 stream/parser/onnx/slice.py create mode 100644 stream/workload/dependency_propagation/slice_node.py diff --git a/.gitignore b/.gitignore index adb0043f..a4f8f6de 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.out typings +outputs # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/outputs/custom_ssm.onnx b/outputs/custom_ssm.onnx index 669004a5e20ead7c1ce9ea8b18c2113ef3578945..f0003846f46c477ba1fa72252d450eb4220016da 100644 GIT binary patch delta 2111 zcmah}OK%%h6z-jIlkv6V8ILDpdu+#ULK`~`89$PcL@Eg+6{w;w3`n#B%Sl|~sb7i5 zGzFmsv7ks4Fb5Vq>kTT8vOrlN1RMTAcStN~P*;^$A~s0CnXx@}>?BySukXF*eCM3+ z-1FU=|6yL@>|%fAs$Sk$SyK)tC#EJ6bB($LBn?Ak2Hxfb-@mO-gV^X*DypUOMo~Ri zz5-k1D3pm(|0N7gG6D^T!bRpdtdeQCBRk*=avL>pHC@QBzT<^M})~ZMi_yb zU9!JHs9Vjh=qg;cDa>RFZrihblscQfCpRmZ*%XB6-wc0fTXDqMBbU(r>0EU!Tghe0 zHN8^PGjU3mM4wu$6*5bi&G%j2BrfX7e5qP14)-j?A>$R+&FPDa$%Z0Vd_}asd z8f$~F^dka7NC{92q~&cz5aAu(3-??fVjl1c|0}r&zX=OQpTeDedY2%C(WuNzSv|L^ zo~{+(j5`KTi9Box(-Kd)%eh=-HD9dyNP{r+c`*z-!l1z_qsZppQ$&GrH^+X1M6U}G zxFx#a55bNmj_oV%`-FS~S#qwkp-t+wF3(~{+67nE=sGB3=0A)B1zdU> z1K&z#uZ>W?$?55=j%j7E`vP^LJ*6zSCE-uw`~_t<@p#9fg$>an7rFsvXFCC+CdzV% z_BET*lgy8UF=wpv8Q+v;2WDydX`>XvSoeMZ08zm$2YHbS7};snazggv>-LC%tp3Ut(2=dxGDSlV+IAjo{PZk)%omb+J}V*NP8HJrFsCDjGPuZ}zDJ^mSjS1l-_#LN_Jw zQ)q&WflH8$M-#A`t@i3~ht5Q`;}q@+L-3k^fSEGo)z{q>q{z^Ywk$g!BXeW|es>23 zdnb!%D#XR$o;OqA3J>pjlYoE3prH|M{flY|3ge!@?z1?By2sQKXRAKxaYm5-1L5n~ AD*ylh literal 8163 zcmbst%W@mX5leu;viX8mluRQsMXysf0ZRg~eCUyf5Iqi3nJU>-s$w33tcW$aE4++{ zMEepZuDB|dW2$n>f!9>!k`Kr+m;98|GrP0XvpWl>OjdER+uhUM)350s)3i#%ZzhA$ z(Me_AxwCtx`QX0~;hFVby+0WB>W>Hg@x<*<+Jot2IGwZ`l@%3Xn}Vda2J>&-?$oQ9 zW#f4vm&>)zjRHJ@dB2ZKA&aABg&bcF-bpD7G0e^*0w$oLvfdA=zVJHJBX8ex`;tL9 ziLybGp{fP<)El>c5P!zGQL3$i`Ns2x9ltkTN5CBH3khP6?Pf}XL}i-dpa1#fE;Gen z4anA5pd`Ool(5jRI-M9+i7bRLJCBc=1&i-I|M+Cm9{b;V>n7^YHsl^r01`!x(0&JBkBIL<0__4rX)NSO6(b+SJ908Dl*^qxWEOgrvVvp>I+cWJp7?L2 z+{iQtv-9|Xhav={7^1|2!Tz+HVt_C^j}LezOhl<*k!8r8(CF4$Xc|U9Vq}p(Lsl3A zT2fdY^!uj|9_+gl#^e63wO;r8?crz;)FW7&f~yIPT?{FHd<+~|E1~#_DkAX}|W}v9(;MAzvx5&r2rLJy4lqcpVJBnhd+sF%2&P zvrX{^FOrHjF$Z6b-2Qkt7<+OtNT9tC<1y6>LvPe;@sTUcE+uIE6+YxpfsM0{0>%PuJdm|uYq-q?1ZG(5(Uq+##7`h zjUibeXRMI*1OK=;@JV%DqF}oe;o)Tl8wBgvSXVfhR*i}&Kw&+2iV=baO&qcdF^uG1 zMdaR!X<+V|2FbN&Sb$b{-#qqzVp6UwFrq6hG}eKHwK+Z>|*WLc@U*b#u+{nY944BCqjgZ_M_AFePN2e)8Ruh<5 ziJL@;GLwy3oD8`dL$1ePSYoIi$G##%j)ODX`FiHgq)te~-6{}L`xlSY#iVwLNUr@aH}-%#J=UUzXq`lkKlRBh~*75F7Pyf0}0?8Q3KGi{FqK=KGk))&!0v#@x0=9&zWbi}E@O#xlVapZ zE)98IPw->r_g`BdP-dy~zY>mO(ar>#c#otnUTOUM?)SUQPyaCBqFumj!Kz7>DMT^p z-nsaIGLUTSD2M%4G~qX! z4Ri|1)6f0oBves3&g2-jIvWgT2{ud4_>V&%p3rI81 zX&8=P;Sk)~;YX0VkSPNWUKX&mYxp?-{O~8@eB>A<*PE(p=3&;M!Z3G{Q z6{3EU3ouKlP!%dD1xR>I;a9@~z_nE>(*G-4=2lv^a_^TRJF*n>S)BKJ!{5SFnd_+* zn@qcA-i7H_oPQqNP{%*V;n7v%(Vq`KNtY*`nps;kJR{V>Q>XE!>Nzs}OH&mvvUKAzTt|H^JPo4rjvWnnvKYaO*|WEG2X5iYEikR( zwEw2vZp3fR_|1vmwu4tQM&^64hrA^9PP}e6e%p)R?#6HTg5O>=$67y!2gqBlz7Gpl ziEG?!$k(QPb>!=Iut8IT%)#tbDdNHQGCk~GQsRs3sS_>Kz`fj(3hv0)UHQ5vU+)IZ ztXXwE)Adf)6li6yQECU75M>1B16F3!6JJKUHq@~}$rZI0hLvI~*t>8h(j||9=hFqF zl4v37s!A6(5~7vS&MKOOH`04Zj~Uu^PP)e@1zc}0%^FhB^=q0lx^=B1*B;H4%l`v% CE+ "list[NodeTensor]": axis = axis - 1 if axis < 0 else axis return [t.view(NodeTensor) for t in np.split(self.as_ndarray(), split_indices, axis=axis)] + def slice(self, starts: int, ends: int, axis: int, steps: int) -> "NodeTensor": + assert starts != 1 and ends != -1 + axis = len(self.tensor_shape) - 1 if axis < 0 else axis + match axis: + case 0: + return self.as_ndarray()[starts:ends:steps, ...].view(NodeTensor) + case 1: + return self.as_ndarray()[:, starts:ends:steps, ...].view(NodeTensor) + case 2: + return self.as_ndarray()[:, :, starts:ends:steps, ...].view(NodeTensor) + case 3: + return self.as_ndarray()[:, :, :, starts:ends:steps, ...].view(NodeTensor) + case _: + raise NotImplementedError + def concat_with_empty(self, shape: tuple[int, ...], axis: int, variable_input_first: bool): empty_shape = self.convert_to_full_shape(shape) empty_tensor = np.zeros(empty_shape, dtype=object) diff --git a/stream/onnx_utils.py b/stream/onnx_utils.py index 150d3701..dfa9b434 100644 --- a/stream/onnx_utils.py +++ b/stream/onnx_utils.py @@ -1,9 +1,7 @@ +import numpy as np from onnx import AttributeProto, ModelProto, NodeProto, numpy_helper from zigzag.parser.onnx.utils import get_onnx_tensor_type -import numpy as np -import onnx - def get_attribute_as_ints( node: NodeProto, attribute_name: str, default: list[int] | int | None = None @@ -60,6 +58,25 @@ def has_asymmetric_input_data(node: NodeProto, onnx_model: ModelProto): return input_shape1 != input_shape2 +def get_constant_tensor_int(onnx_model: ModelProto, constant_output_name: str): + """In some cases, the constants to a node (e.g. slice and split indices) are saved as tensors within a constant + node. The output name of the constant nodes corresponds to the input name of the node that uses this constant + tensor.""" + + for node in onnx_model.graph.node: + if node.op_type == "Constant" and node.output[0] == constant_output_name: + for attr in node.attribute: + if attr.name == "value": + tensor = attr.t # This is an ONNX TensorProto + # Decode tensor to a numpy array + array = np.frombuffer(tensor.raw_data, dtype=int) + array = array.reshape([dim for dim in tensor.dims]) + + return [int(i) for i in array] + + raise ValueError(f"Cannot find {constant_output_name}") + + def get_axis_attribute(node: NodeProto): """Find the value of the axis associated with this ONNX node""" ATTR_NAME = "axis" @@ -71,19 +88,20 @@ def get_axis_attribute(node: NodeProto): def get_split_attribute(node: NodeProto, onnx_model: ModelProto): - # ATTR_NAME = "split" - output_name = next(n for n in node.input if "split" in n.lower()) + return get_constant_tensor_int(onnx_model, output_name) - for node in onnx_model.graph.node: - if node.op_type == "Constant" and node.output[0] == output_name: - for attr in node.attribute: - if attr.name == "value": - tensor = attr.t # This is an ONNX TensorProto - # Decode tensor to a numpy array - array = np.frombuffer(tensor.raw_data, dtype=int) - array = array.reshape([dim for dim in tensor.dims]) - return [int(i) for i in array] +def get_slice_attributes(node: NodeProto, onnx_model: ModelProto): + """Get the `starts`, `ends`, `axes` and `steps` tensors for a slice node. + NOTE: this assumes that the attributes are given as inputs in this order""" + if len(node.input) != 5: + raise NotImplementedError("Unsure how to get slice attributes from Node") + + starts_output_name, ends_output_name, axes_output_name, steps_output_name = node.input[1:5] - raise ValueError + starts_value = get_constant_tensor_int(onnx_model, starts_output_name) + ends_value = get_constant_tensor_int(onnx_model, ends_output_name) + axes_value = get_constant_tensor_int(onnx_model, axes_output_name) + steps_value = get_constant_tensor_int(onnx_model, steps_output_name) + return starts_value, ends_value, axes_value, steps_value diff --git a/stream/parser/onnx/conv.py b/stream/parser/onnx/conv.py index 10f8566e..af53bb26 100644 --- a/stream/parser/onnx/conv.py +++ b/stream/parser/onnx/conv.py @@ -26,6 +26,8 @@ def get_layer_node_user_format( ) -> dict[str, Any]: """ Generate the necessary dictionary items required for the LayerNode creation. + + """ predecessors = self.get_node_predecessors() @@ -37,67 +39,93 @@ def get_layer_node_user_format( group_size: int = get_attribute_ints_with_name("group", attrs, default=1) # type:ignore padding: list[int] = get_attribute_ints_with_name("pads", attrs, default=[0, 0, 0, 0]) # type:ignore - # 1D Conv case: append dimensions of size 1 so equation holds. Conv in FY dimension - print(kernel_shape) - if len(kernel_shape) == 1: - kernel_shape.insert(0, 1) - input_shape.append(1) - output_shape.append(1) - strides.append(1) - dilations.append(1) - assert len(input_shape) == 4 - assert len(output_shape) == 4 - - if len(padding) == 2: - padding = 2 * padding - data: dict[str, Any] = {} data["id"] = self.node_id data["name"] = self.node.name data["operator_type"] = ConvParser.OP_TYPE + data["operand_precision"] = self.get_operand_precision_user_format() + data["operand_source"] = self.get_operand_source_user_format(predecessors) - # IMPORTANT: If any of the input loops require padding, they should be defined as the rightmost dimensions in - # the equation. This is because we construct the dimensionality order and then add the padding to those last - # dimensions in the order - weight_dim = "g" if group_size > 1 else "k" - data["equation"] = f"O[b][g][k][oy][ox]+=W[{weight_dim}][c][fy][fx]*I[b][g][c][iy][ix]" + # 1D Conv case: append dimensions of size 1 so equation holds. Conv in FY dimension + is_1d_conv = len(kernel_shape) == 1 + + # if len(kernel_shape) == 1: + # kernel_shape.insert(0, 1) + # input_shape.append(1) + # output_shape.append(1) + # strides.append(1) + # dilations.append(1) + # assert len(input_shape) == 4 + # assert len(output_shape) == 4 + + # if len(padding) == 2: + # padding = 2 * padding # Get dimension sizes from input parameters assert input_shape[0] == output_shape[0], "Batch size is different for input and output activations." B = output_shape[0] G = group_size K = ceil(output_shape[1] / G) - OX = output_shape[3] - OY = output_shape[2] C = ceil(input_shape[1] / G) - IX = input_shape[3] - IY = input_shape[2] FX = kernel_shape[0] - FY = kernel_shape[1] - data["loop_dims"] = ["B", "K", "G", "OX", "OY", "C", "FX", "FY"] - data["loop_sizes"] = [B, K, G, OX, OY, C, FX, FY] - - data["pr_loop_dims"] = ["IX", "IY"] - data["pr_loop_sizes"] = [IX, IY] - data["dimension_relations"] = [ - f"ix={strides[0]}*ox+{dilations[0]}*fx", - f"iy={strides[1]}*oy+{dilations[1]}*fy", - ] - data["operand_precision"] = self.get_operand_precision_user_format() - data["operand_source"] = self.get_operand_source_user_format(predecessors) + IX = input_shape[2] + OX = output_shape[2] - # Add information wrt how this conv node's input/output tensors - # are represented in the onnx model vs how they are represented in the equation above. - # Because onnx doesn't actually encode the group dimension in a separate dimension - # but instead keeps it as a "groups" parameter. - # Concretely, this entry contains for the I and O operand how the G + C/K should be converted - # to a single "CH" (channel) dimension. - - # Add padding information - data["padding"] = [ - [padding[0], padding[2]], - [padding[1], padding[3]], - ] + weight_dim = "g" if group_size > 1 else "k" + + # IMPORTANT: If any of the input loops require padding, they should be defined as the rightmost dimensions in + # the equation. This is because we construct the dimensionality order and then add the padding to those last + # dimensions in the order. + # Add information wrt how this conv node's input/output tensors are represented in the onnx model vs how they + # are represented in the equation. Because onnx doesn't actually encode the group dimension in a separate + # dimension but instead keeps it as a "groups" parameter. Concretely, this entry contains for the I and O + # operand how the G + C/K should be converted to a single "CH" (channel) dimension. + + if is_1d_conv: + # No FY, OY, IY + data["loop_sizes"] = [B, K, G, OX, C, FX] + data["loop_dims"] = ["B", "K", "G", "OX", "C", "FX"] + data["equation"] = f"O[b][g][k][ox]+=W[{weight_dim}][c][fx]*I[b][g][c][ix]" + data["pr_loop_dims"] = ["IX"] + data["pr_loop_sizes"] = [IX] + data["dimension_relations"] = [ + f"ix={strides[0]}*ox+{dilations[0]}*fx", + ] + data["padding"] = [ + [padding[0], padding[1]], + ] + else: + assert len(input_shape) == 4 and len(output_shape) == 4 and len(padding) == 4 and len(strides) == 2 + FY = kernel_shape[1] # TODO is kernel_shape in (FX, FY) format or (FY, FX)? (I assumed the former) + IY = input_shape[3] + OY = output_shape[3] + data["loop_sizes"] = [B, K, G, OX, C, FX, OY, FY] + data["loop_dims"] = ["B", "K", "G", "OX", "C", "FX", "OY", "FY"] + data["equation"] = f"O[b][g][k][oy][ox]+=W[{weight_dim}][c][fy][fx]*I[b][g][c][iy][ix]" + data["pr_loop_dims"] = ["IX", "IY"] + data["pr_loop_sizes"] = [IX, IY] + data["dimension_relations"] = [ + f"ix={strides[0]}*ox+{dilations[0]}*fx", + f"iy={strides[1]}*oy+{dilations[1]}*fy", + ] + data["padding"] = [ + [padding[0], padding[2]], + [padding[1], padding[3]], + ] + + # Remove dims with size 1 + dims_size_1 = [dim for dim, size in zip(data["loop_dims"], data["loop_sizes"]) if size == 1] + data["loop_sizes"] = [s for s in data["loop_sizes"] if s > 1] + data["loop_dims"] = [d for d in data["loop_dims"] if d not in dims_size_1] + for dim in dims_size_1: + data["equation"] = data["equation"].replace(f"[{dim.lower()}]", "") + + # Filter out loops with size 1 + # loop_sizes = {"B": B, "K": K, "G": G, "OX": OX, "OY": OY, "C": C, "FX": FX, "FY": FY} + # dims_with_size_1 = [k for k, v in loop_sizes.items() if v == 1] + # loop_sizes = {k: v for k, v in loop_sizes.items() if v > 1} + # data["loop_dims"] = list(loop_sizes.keys()) + # data["loop_sizes"] = list(loop_sizes.values()) return data diff --git a/stream/parser/onnx/einsum.py b/stream/parser/onnx/einsum.py index 003ed5ab..1cdc53f0 100644 --- a/stream/parser/onnx/einsum.py +++ b/stream/parser/onnx/einsum.py @@ -15,8 +15,9 @@ def get_einsum_equation(self): attrs_names = [attr.name for attr in self.node.attribute] name_idx = attrs_names.index(ATTR_NAME) - value = self.node.attribute[name_idx] - return str(value) + attr_proto = self.node.attribute[name_idx] + value = attr_proto.s.decode("utf-8") + return value def get_layer_dims_per_op(self): einsum_equation = self.get_einsum_equation() @@ -32,7 +33,7 @@ def put_in_brackets(s: str): raise NotImplementedError dims_I, dims_W, dims_O = layer_dims_per_op - equation = f"O{put_in_brackets(dims_O)}+=I{put_in_brackets(dims_I)}*{put_in_brackets(dims_W)}" + equation = f"O{put_in_brackets(dims_O)}+=I{put_in_brackets(dims_I)}*W{put_in_brackets(dims_W)}" return equation # def get_layer_dims(self, layer_dims_per_op: list[str]): diff --git a/stream/parser/onnx/model.py b/stream/parser/onnx/model.py index a648fba8..21a34f0b 100644 --- a/stream/parser/onnx/model.py +++ b/stream/parser/onnx/model.py @@ -10,6 +10,7 @@ from stream.parser.onnx.concat import ConcatParser from stream.parser.onnx.conv import ConvParser from stream.parser.onnx.default import DefaultNodeParser +from stream.parser.onnx.einsum import EinsumParser from stream.parser.onnx.flatten import FlattenParser from stream.parser.onnx.gather import GatherParser from stream.parser.onnx.gemm import GemmParser @@ -19,6 +20,7 @@ from stream.parser.onnx.pooling import PoolingParser from stream.parser.onnx.reshape import ReshapeParser from stream.parser.onnx.simd import SimdParser +from stream.parser.onnx.slice import SliceParser from stream.parser.onnx.softmax import SoftmaxParser from stream.parser.onnx.split import SplitParser from stream.parser.onnx.transpose import TransposeParser @@ -37,6 +39,7 @@ class ONNXModelParser: "Conv": ConvParser, "MatMul": MatMulParser, "Gemm": GemmParser, + "Einsum": EinsumParser, "MaxPool": PoolingParser, "AveragePool": PoolingParser, "GlobalMaxPool": PoolingParser, @@ -44,6 +47,7 @@ class ONNXModelParser: "Add": SimdParser, "Mul": SimdParser, "Softmax": SoftmaxParser, + # Activations "Relu": SimdParser, "Gelu": SimdParser, "Silu": SimdParser, @@ -55,6 +59,7 @@ class ONNXModelParser: "Flatten": FlattenParser, "Concat": ConcatParser, "Split": SplitParser, + "Slice": SliceParser, } def __init__( diff --git a/stream/parser/onnx/slice.py b/stream/parser/onnx/slice.py new file mode 100644 index 00000000..113b5d8e --- /dev/null +++ b/stream/parser/onnx/slice.py @@ -0,0 +1,33 @@ +from stream.onnx_utils import get_slice_attributes +from stream.parser.onnx.operator_parser import OnnxOperatorParser +from stream.workload.dependency_propagation.slice_node import SliceNode + + +class SliceParser(OnnxOperatorParser): + """Parses an onnx gather operator into a SliceNode.""" + + def generate_node(self): + if len(self.node.output) > 1: + raise NotImplementedError("Slice node with multiple output slices not yet supported.") + + # Single predecessor + predecessors = self.get_node_predecessors() + if len(predecessors) > 1: + raise ValueError("Slice node should not have more than one input") + predecessor = predecessors.pop() + + starts_value, ends_value, axes_value, steps_value = get_slice_attributes(self.node, self.onnx_model) + input_names = list(self.node.input) + output_names = list(self.node.output) + + return SliceNode( + node_id=self.node_id, + node_name=self.node.name, + predecessor=predecessor, + starts=starts_value, + ends=ends_value, + axes=axes_value, + steps=steps_value, + input_names=input_names, + output_names=output_names, + ) diff --git a/stream/stages/generation/tiled_workload_generation.py b/stream/stages/generation/tiled_workload_generation.py index 92853964..88479341 100644 --- a/stream/stages/generation/tiled_workload_generation.py +++ b/stream/stages/generation/tiled_workload_generation.py @@ -20,7 +20,6 @@ from stream.workload.dependency_propagation.concat_node import ConcatNode from stream.workload.dependency_propagation.dummy_node import DummyNode from stream.workload.dependency_propagation.propagation_node import PropagationNode -from stream.workload.dnn_workload import DNNWorkloadStream from stream.workload.node import Node from stream.workload.onnx_workload import ComputationNodeWorkload, ONNXWorkload from stream.workload.tensor import Tensor @@ -123,7 +122,7 @@ def get_scheduling_order(workload: ComputationNodeWorkload): return sorted(((n.id, n.sub_id) for n in workload.node_list), reverse=True) @staticmethod - def get_all_node_pairs(G: DNNWorkloadStream) -> tuple[tuple[ComputationNode, ComputationNode, bool], ...]: + def get_all_node_pairs(G: ONNXWorkload) -> tuple[tuple[ComputationNode, ComputationNode, bool], ...]: pairs: list[tuple[ComputationNode, ComputationNode, bool]] = [] for node in G.topological_sort(): if not isinstance(node, ComputationNode): diff --git a/stream/stages/generation/tiling_generation.py b/stream/stages/generation/tiling_generation.py index 829c8b13..d0a463fb 100644 --- a/stream/stages/generation/tiling_generation.py +++ b/stream/stages/generation/tiling_generation.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict from typing import Any import numpy as np @@ -16,6 +17,31 @@ class TilingGenerationStage(Stage): + # Split the node in this dimension to enable fusion within core + FUSION_PARTITION_DIM_DEFAULT: defaultdict[str, LayerDim] = defaultdict( + lambda: LayerDim("K"), + { + "conv": LayerDim("OY"), + "matmul": LayerDim("D"), + "gemm": LayerDim("D"), + "pooling": LayerDim("OY"), + "add": LayerDim("D"), + "mul": LayerDim("D"), + "softmax": LayerDim("K"), + "max": LayerDim("K"), + "div": LayerDim("K"), + "exp": LayerDim("K"), + "sum": LayerDim("K"), + "relu": LayerDim("K"), + "gelu": LayerDim("K"), + "silu": LayerDim("K"), + }, + ) + FUSION_PARTITION_SIZE_DEFAULT = 2 + + # Split node in this dimension to partition layer over cores. NOTE this list is ordered + INTER_CORE_PARTITION_DIM_DEFAULT = [LayerDim("G"), LayerDim("H"), LayerDim("K")] + def __init__( self, list_of_callables: list[StageCallable], @@ -109,11 +135,22 @@ def remove_invalid_entries_from_intra_core_tiling(self, node: ComputationNode): node.intra_core_tiling = valid_tiling - def generate_intra_core_tiling(self, node: ComputationNode) -> TILING_T: - partition_dim = node.fusion_partition_dims[0] + def get_fusion_partition_dim(self, node: ComputationNode) -> LayerDim: + partition_dim = TilingGenerationStage.FUSION_PARTITION_DIM_DEFAULT[node.type] + + # Default partition dim is not present in this node -> take some arbitrary other dim if partition_dim not in node.layer_dim_sizes: - raise ValueError(f"Suggested partition dimension {partition_dim} for {node} is not part of this node") - return [(node.fusion_partition_dims[0], node.layer_dim_sizes[partition_dim])] + partition_dim: LayerDim = next( + dim for dim in node.layer_dim_sizes if dim != LayerDim("B") and dim != LayerDim("G") + ) + + return partition_dim + + def generate_intra_core_tiling(self, node: ComputationNode) -> TILING_T: + partition_dim = self.get_fusion_partition_dim(node) + size = min(TilingGenerationStage.FUSION_PARTITION_SIZE_DEFAULT, node.layer_dim_sizes[partition_dim]) + tiling = [(partition_dim, size)] + return tiling def remove_invalid_entries_from_inter_core_tiling(self, node: ComputationNode): """Check wether this node's inter core tiling has invalid entries: non-existent layer dimension for this node @@ -143,14 +180,11 @@ def remove_invalid_entries_from_inter_core_tiling(self, node: ComputationNode): node.inter_core_tiling = valid_tiling def generate_inter_core_tiling(self, node: ComputationNode) -> TILING_T: - if node.layer_dim_sizes.data.get(LayerDim("G"), 1) > 1: - loop_dim = LayerDim("G") - elif node.layer_dim_sizes.data.get(LayerDim("K"), 1) > 1: - loop_dim = LayerDim("K") - else: - raise ValueError("Unknown what loop dim to split across cores") - - return [(loop_dim, "*")] + for dim in TilingGenerationStage.INTER_CORE_PARTITION_DIM_DEFAULT: + if dim in node.layer_dim_sizes and node.layer_dim_sizes[dim] > 1: + return [(dim, "*")] + + raise ValueError("Unknown what loop dim to split across cores") @staticmethod def split_operator(model: ModelProto, node_name: str, num_splits: int): diff --git a/stream/workload/computation/computation_node.py b/stream/workload/computation/computation_node.py index 58e7e535..2cdc51af 100644 --- a/stream/workload/computation/computation_node.py +++ b/stream/workload/computation/computation_node.py @@ -32,24 +32,6 @@ class ComputationNode(LayerNode, Node): too_large_operands: list[MemoryOperand] - # Map the node's op_type to the corresponding layer dimension to split on for fusion - FUSION_DIM_MAPPING: dict[str, list[LayerDim]] = { - "conv": [LayerDim("OY")], - "matmul": [LayerDim("D")], - "gemm": [LayerDim("D")], - "pooling": [LayerDim("OY")], - "add": [LayerDim("D")], - "mul": [LayerDim("D")], - "softmax": [LayerDim("K")], - "max": [LayerDim("K")], - "div": [LayerDim("K")], - "exp": [LayerDim("K")], - "sum": [LayerDim("K")], - "relu": [LayerDim("K")], - "gelu": [LayerDim("K")], - "silu": [LayerDim("K")], - } # TODO default to "K" ? - def __init__( self, node_id: int, @@ -113,11 +95,6 @@ def __init__( self.nb_real_predecessors = None self._static_hash_value = self.__compute_static_hash() - try: - self.fusion_partition_dims = ComputationNode.FUSION_DIM_MAPPING[op_type] - except KeyError: - raise NotImplementedError(f"Fusion partitioning dimensions not defined for {op_type}") - # Each ComputationNode will save a tensor for all its defined operands. # For example, a conv layer will have an I tensor, W tensor and O tensor. self.operand_tensors: dict[LayerOperand, Tensor] = {} diff --git a/stream/workload/dependency_propagation/slice_node.py b/stream/workload/dependency_propagation/slice_node.py new file mode 100644 index 00000000..49de89d5 --- /dev/null +++ b/stream/workload/dependency_propagation/slice_node.py @@ -0,0 +1,45 @@ +from zigzag.datatypes import Constants + +from stream.node_tensor import NodeTensor +from stream.workload.dependency_propagation.propagation_node import PropagationNode +from stream.workload.node import Node + + +class SliceNode(PropagationNode): + """Class that represents an onnx Slice node.""" + + def __init__( + self, + node_id: int, + node_name: str, + predecessor: int, + starts: list[int], + ends: list[int], + axes: list[int], + steps: list[int], + output_names: list[str], + input_names: list[str] = [], + ) -> None: + """Initialize the SliceNode + Slice the tensor at axis `axis`. The sizes are given by `Slices`. `len(Slices)` is the number of output nodes. + + Args: + predecessors: The id of this node's parent. + axis: axis in which to Slice + Slices: sizes of the output Slices in the given axis + output_names: the node names that correspond to the Slices + """ + op_type = "Slice" + super().__init__(node_id, node_name, op_type, input_names) + + self.starts = starts + self.ends = ends + self.axes = axes + self.steps = steps + self.input_operand_source = {Constants.LAYER_OP_I: predecessor} + self.output_names = output_names + + def propagate(self, tensor: NodeTensor, next_node: Node | None = None): + """Slice the tensor. + Currently assumes only one slice is created.""" + return tensor.slice(starts=self.starts[0], ends=self.ends[0], axis=self.axes[0], steps=self.steps[0]) From e27c2a8b8137ba498ac4f122a36da1e19dbe49cd Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Thu, 7 Nov 2024 16:22:51 +0100 Subject: [PATCH 4/8] Create MulParser that deals with asymmetrical inputs and allows broadcasting --- outputs/custom_ssm.onnx | Bin 8121 -> 10998009 bytes stream/parser/onnx/einsum.py | 19 +++-- stream/parser/onnx/model.py | 24 +++--- stream/parser/onnx/mul.py | 73 ++++++++++++++++++ stream/parser/onnx/simd.py | 1 + .../generation/tiled_workload_generation.py | 8 +- stream/stages/generation/tiling_generation.py | 3 +- 7 files changed, 105 insertions(+), 23 deletions(-) create mode 100644 stream/parser/onnx/mul.py diff --git a/outputs/custom_ssm.onnx b/outputs/custom_ssm.onnx index f0003846f46c477ba1fa72252d450eb4220016da..97da8bdb06facb9bb63f254d0aadd3a0dffa129e 100644 GIT binary patch literal 10998009 zcmeFw!A=uV6adiH(l87OG^2^wmFdQ$3lI#^-~x=ekc7mD`=+52X@n`#7Q|IQ!Nd;$ z_x^@EzrkP8sdPGp!XKPjyiD(Xz3;yB-UM^e?D2W8bFzO>Ew3%FFRyO=`EgMW-$oBt zlFrG|%Ii+j?KP8Lqtovl_j`?})rIScT98etKM2dG&BK0N4=Tl-nL?qkby=JVx5IKf zN&gz&RWXXLD!%KS<*Uq%W3^J62=}5#*Vk>V=94QoQxi>9v!)h;Vr6zG_&a)rt(}Lr zEFIpxP}alyxYgf}cjIO<)kB`Dh537ZF$_M%@o}qt)LklG7Vm|-S*=X3u{KpJPfhIS zI`Yq7Po58V^1Ya~oi94Bm!a_MgpG@4UFb~#dhN)YGp5-mxJn?6D3Ka^2 z&7shTDTTaTcw0*u&35}VUw1^4#%iT>2sg9KJ-SGfld}T5{llpOX{=UChw%4cP>vQm zN%Cc5<4rs1_KzB?FRJtTQLRtx>>{0!M2oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkL{A3Jm)000000ObGL2nh}xIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB) z95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c z2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*= zfddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede z;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQq zIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n? z4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj! z0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^` zz<~n?4jede;J|?c2M!!KaNxj!0|yQqIB?*=fddB)95`^`z<~n?4jede;J|?c2M!!K zaNxj!1KP2>-nNqS0KldtTlDxt^0C%Yl#Qb@i^f*Gr6qZ@seu-o%?1b-aM4{~t$V@H z5@VB;4xrHoHJ)Wo_Xgz zb4Y*y0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7csf&Vjsw~giSI7!;S91Ty#-Ql=1IvpRMjyn%FujFN|W_ESE z6)t_#Jv{Ag-)>%9TlsvgnGHNl|45trO-S0q(aWRur@0gbH!iGC<=)JGZ_Pwg$-i%eGv^R${UxjI zY1i2#>jgXYg5Ax_(_31XCUjGgvww|rW?v=i#nSB*D|NM0dd~c^2+8Kx zD9vv_)CRp17WS3g%A%rE8bBxt&kL6S`tDA72D=Mquv?nJ@4mZ}tH?F=57tw_eaBT zb`JjJRj>d2#W?wX`{?wrla{rA)gPXm9u=Le)i1R!PpfSIAT)blpLWOn(Xe&x*}<#( z2Zygd{PXz7+s0BDhO^sD-fzF`9h~m>7Wa5-!PUIe+oeudVqa;8X@7qg&TTYYolW>z zzk8BgZ~x-ppqS!v`mwbXe{3h=>Xf8&(*L%%(TslEtD)K5AD0SZrP)$VDy^l+e-^5Z zChxW5P3_J^Ti-wAYz1{9c)4%C__IiSw=gaGr@^=*eqMRun z=ACbrGR4DErf6jkpWDrg@!_+UWs2+}lsE>$f$~5QxRr14fdG-JkF`jUKzsZlvH-0%ekv;BphsD5dFRsuEMdkE#^w+)K@j?ISWMgvN{Hoe+KHbtY zrqXUvp)}?7Y|72G#^=lb`SCw}luq_P8sQI(xmuxUJD*f_`O<8YZ-~nB#VhZ)B_3B- zrrcFLaF^EX$=5GSr~0+}rPgv)&i%5lRL zPukKZsF7#YbGKx(Jg)R~QCjMobJwPuq8rY+t9~SVFyBRy$v5Y^h);_+itxL}d>_Sa z*%R?`cbpFSX9=@0 zJ5pT;(?8N}aug%wtf`c9XVGgZcNPh?lsg}umpi{Mg-vmHxYQrUx7p-lvBgViPxqewxE@jQKoMfbj@_gq zi9gk`*y@Z;=;JVO^d zGau>QY^0Ucxv05&F$5=bs-)6nQPugLi&Qt?^Jo2U&YwuGnI@6kdg0n^X8|%hy6yZt z`CfQ0(%hWUY^5>Mh#x)~Y{jYPG0Svf8kDUgllow>8?%x zZ^dRxubm4+ZfUSx$~vVgW%+FDMuk#)r9fLyd1>&I`LD|=yJObs^58*?;iI_f`wPA{ zTYN|R{4c5It+qn#Tcw)#y}{kHzA2iknFdVyl~g5A=W5krVDD#@7)aZwzqHp!eK&6I zxmr85HU4B#rQNwoKZx_Me2-JgmxJ_TdmA*)n29RRVwjZN;k!RuEz5jAsV;FBtr~#&;L@PmiM+M8D($whIlalt-{%9X!p!_ z__ObcX|DDXe|Ft;uhm{;D?2BjaeFrUMen!eU7X+W;aM+_i`%cPP;rPU1^>mtAD1#t zX%TB%=`!OU@7Y^$^|4#8JU8t5SBUy8AI=}$OCz3BTu{BVhvur^kE4&z%>T(_F+M{pu~9lR W7b{cUy`}J}hUWB0`su~B_5TKE)quPJ literal 8121 zcmbst%W~UBk`O5oYDp#yOO8T24i`>hk?R;HCHa-gnz7|WT%}ZO{)Nia@!3-Tu!5iQWdV$1=A=O zpPIkMe@wMEpb-t(g18RPz0UN&+x6VOqJu!$Ab!_NV1Dg+!;aq@uN$Yv9PDyl+-Y*-m~kxY|N!!jGc_zXZi?|4FBlz$&rr5oYJ{QMX{}wdWrmP1`x0}4Q*I7C9CGTY=w+`g2(bt&@J-Zvu>RUm+NfuS@O+7v3LJ-(S~pegWHIbL21zM zA3u1o>rNQ>-aTvG@%!!JXb?CtEXl`G3S%!s6#u{5_N)t$_`PA*pR}8e%9043JH*m; zo?F+TVd#x|tz2uag&Qxz{|t=r1fQ&jm4iY5Rikq& z%5PYTKSP78Inft814s%iQ?OlN$a)zTJAT)l_;^}`0_!UfUUhwCRbRxU2V^4H#| z?{(YbBX{Vv3aAv)6ozhR+{(A|?DGN?$CHuY@gf%RCFnJ<*2qrCDX>hzGp+Fi9f5kM z4z|SvMnXJ&d>AC8KebkqCG#6NNkCXNXRwN+{I4w0B&|e^s}NSSTF%o~6kAhDp(0D_u75zn8vjPdYPVSpRdF|aO%U|N$aCI>~8J5?pOHc6G-x=oywRV7=o|M{Ca?|TDO$#Qwz z+C=4|@+cglf*LBQiCc&l&MuQ1upAu#PmYJ}`#Ebki$cfiPTck*=ZW7RPkTbxB@NbIh^XpU!SueFhQ-@n zdD%U>`Lf&p@K5-oe`@66ZwAcdjYdfC4KFL9GLMcGMM^15qQtvIi87OoS{w~SHHM)c zKNy(1vd8g4ks-&r=1hkjcjC3N#!tMlqQ{yRpQ%l#Ok|Wuo5wV~wuPZdk=-WYMK0|N zPYdh}Vafgi&QOy^lMrH+oyXJO9`$cAhS&=c297}?QlqsQJL;(YSTRuKD1k{#bB#Ev z#C*g&6`O{|%U7C!0AnW>wbRau34uvOT_K{DnW&pbQ9v*`|6=z^=6<1L;)WO_SSA?j zvg2Z_^QOIXEMNClB=|Y+eJ^Nq+)0Gf;xp?Soi0B7TFqi6fQ%i`SP>%e*2uGyR7w!Q zS=DDB?~LkdEfC$y#s%#|%(jkHrz@otCMxHDw%(y|<%UpM)uBmBrSC2LqR))<4;C94 z`oL_t*1L3}rkSo>))UNb_g~0-lkF0xNPGnW*)pjjIA8hs?%%iBX8&WrSvt!Ef_D;` zviqbyNOIr#TcTarGgMSO5s70+9uGwQK0(Dpg1^LFJ;8`grOa~RWB+{Ml4e{o`E!pG zFjw;5m~h%8IE%Mcp<%L%b&9z9qJoQ(!i5>D(jDP#RfM>7R}>M=NW{MlRfbpw^4#J6 zee?-l`}Bi{avF91$-`@OXO_Hp(qb({_t46UGxElMXXF;Bb8)_MIC6(a`&YT7 zOK{ONtQD9uP5duG-aJ{|UlS7p02A>A#8vl8?2My;_foyJ#knDu8ZI&Lzp!7CTka)T zM2_=JQf>cw${RQZDg#b2Q4G@rAE(cdfqYe`Dx?{pI|+l0kju+=AU0^CAl+{8hwQQK z0u)f0Tu|c^EIPo|bGxAo;Y3!1)+>hQGy;bcYAUd0K^X-vqX8qopfQ8%d_hv(L{wW` z(+#evRCaXiYMAxm{97tRzf&^lXcfuwfKaoM#G0&T@Rmz&O$Tdf##B(V;PhE=RqiC4 z4Px_r@PS%kc+7H3W87`1bU2$*Go=>1pXS%p*sqG7OApBut5gH#Ii(NMva6%##K=Nx zz&Ha-!Og4*6WvctZH3f=GYMQuJ!e{B_6AkdIElqX4VIof#%+3KBkVG1NJMASkl3R% zp>L(xir#0;BHDR~yvMK>*3c{JutLu_6?r(r5ZdKKKIu+c$7dtBp>{D1O&GLvgu3u+ z1y?ZqqzML7EUxMyQkqXtBnwk7P$Yu2PSOc(-M!N+HHv$L09quD^3 zT~(8LI~fq8Em{9F&u4x=(BAviJk)G~U+~2lt6SDk eZ_7k`O;OwkG)qX~)-Jg{ `[a][b][c]""" + if s == "": + return "[]" return "".join([f"[{char}]" for char in s]) - if len(layer_dims_per_op) != 3: - raise NotImplementedError + match len(layer_dims_per_op): + case 2: + dims_I, dims_O = layer_dims_per_op + dims_W = "" + case 3: + dims_I, dims_W, dims_O = layer_dims_per_op + case _: + raise NotImplementedError - dims_I, dims_W, dims_O = layer_dims_per_op equation = f"O{put_in_brackets(dims_O)}+=I{put_in_brackets(dims_I)}*W{put_in_brackets(dims_W)}" return equation - # def get_layer_dims(self, layer_dims_per_op: list[str]): - # all_dims = {char.upper() for group in layer_dims_per_op for char in group} - # return list(all_dims) - def get_layer_dim_sizes_dict(self, layer_dims_per_op: list[str]): input_shapes = get_onnx_input_shapes(self.node, self.onnx_model) output_shapes = get_onnx_output_shapes(self.node, self.onnx_model) @@ -71,7 +74,7 @@ def get_layer_node_user_format( input_shape: list[int], # Argument required because of a caller function in superclass output_shape: list[int], # TODO put shape logic in this method for all `OnnxComputeOperatorParser` subclasses ) -> dict[str, Any]: - """! Generate layer data in user input format for Einsum.""" + """Generate layer data in user input format for Einsum.""" predecessors = self.get_node_predecessors() data: dict[str, Any] = {} diff --git a/stream/parser/onnx/model.py b/stream/parser/onnx/model.py index 21a34f0b..7ec7f96a 100644 --- a/stream/parser/onnx/model.py +++ b/stream/parser/onnx/model.py @@ -5,8 +5,6 @@ from zigzag.parser.onnx.utils import parse_onnx_model_from_path from stream.hardware.architecture.accelerator import Accelerator -from stream.onnx_utils import get_onnx_input_shapes, has_asymmetric_input_data -from stream.parser.onnx.asymmetric_simd import AsymmetricSimdParser from stream.parser.onnx.concat import ConcatParser from stream.parser.onnx.conv import ConvParser from stream.parser.onnx.default import DefaultNodeParser @@ -16,6 +14,7 @@ from stream.parser.onnx.gemm import GemmParser from stream.parser.onnx.lpnormalization import LpNormalizationParser from stream.parser.onnx.matmul import MatMulParser +from stream.parser.onnx.mul import MulParser from stream.parser.onnx.operator_parser import OnnxOperatorParser from stream.parser.onnx.pooling import PoolingParser from stream.parser.onnx.reshape import ReshapeParser @@ -44,8 +43,8 @@ class ONNXModelParser: "AveragePool": PoolingParser, "GlobalMaxPool": PoolingParser, "GlobalAveragePool": PoolingParser, - "Add": SimdParser, - "Mul": SimdParser, + "Add": MulParser, + "Mul": MulParser, "Softmax": SoftmaxParser, # Activations "Relu": SimdParser, @@ -79,15 +78,14 @@ def run(self): self.workload = self.parse_workload() def get_parser_class(self, node: NodeProto): - # A temporary fix an element-wise Add or Mul which has asymmetric input data -> treat it as a DummyNode. - # TODO support node with asymmetric input data. - if node.op_type in ["Add", "Mul"] and has_asymmetric_input_data(node, self.onnx_model): - in_shape_1, in_shape_2 = get_onnx_input_shapes(node, self.onnx_model) - # In case only the batch dimension is missing. Other cases are not supported for now - if abs(len(in_shape_1) - len(in_shape_2)) == 1: - return AsymmetricSimdParser - else: - return DefaultNodeParser + # # A temporary fix an element-wise Add which has asymmetric input data -> treat it as a DummyNode. + # if node.op_type in ["Add", "Mul"] and has_asymmetric_input_data(node, self.onnx_model): + # in_shape_1, in_shape_2 = get_onnx_input_shapes(node, self.onnx_model) + # # In case only the batch dimension is missing. Other cases are not supported for now + # if abs(len(in_shape_1) - len(in_shape_2)) == 1: + # return AsymmetricSimdParser + # else: + # return DefaultNodeParser parser_class = ONNXModelParser.OP_TYPE_TO_PARSER.get(node.op_type) if not parser_class: diff --git a/stream/parser/onnx/mul.py b/stream/parser/onnx/mul.py new file mode 100644 index 00000000..1d78fced --- /dev/null +++ b/stream/parser/onnx/mul.py @@ -0,0 +1,73 @@ +from typing import Any + +from stream.onnx_utils import get_onnx_input_shapes, get_onnx_output_shapes +from stream.parser.onnx.operator_parser import OnnxComputeOperatorParser + + +class MulParser(OnnxComputeOperatorParser): + """Parses an ONNX operator representing an elementwise operation (Mul) into a ComputationNode.""" + + def get_common_and_broadcast_shape(self): + """This node assumes that the ONNX node has 2 inputs and 1 output. One input shape is identical to the output + shape, and the other shape can broadcast in dimensions. + Returns the common shape (in and out) and the broadcast shape""" + input_shapes = get_onnx_input_shapes(self.node, self.onnx_model) + output_shapes = get_onnx_output_shapes(self.node, self.onnx_model) + + if len(input_shapes) != 2 or len(output_shapes) != 1: + raise NotImplementedError + + output_shape = output_shapes.pop() + if not any(shape == output_shape for shape in input_shapes): + raise NotImplementedError + + input_shape = output_shape + input_shapes.remove(output_shape) + broadcast_shape = input_shapes.pop() + + # e.g. (3,5) * (8,3,5) is ok (broadcast over dim 0), but (3,2) * (8,3,5) is unclear + for broadcast_size, in_size in zip(reversed(broadcast_shape), reversed(input_shape)): + if broadcast_size != in_size and broadcast_size != 1: + raise ValueError + + return input_shape, broadcast_shape + + def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[int]): + """ + Generate the necessary dictionary items required for the LayerNode creation. + """ + common_shape, broadcast_shape = self.get_common_and_broadcast_shape() + + data: dict[str, Any] = {} + data["id"] = self.node_id + data["name"] = self.node.name + data["operator_type"] = self.node.op_type + data["operand_source"] = self.get_operand_source_input_format() + data["operand_precision"] = self.get_operand_precision_user_format() + data["dimension_relations"] = [] + data["loop_sizes"] = common_shape + + match len(common_shape): + case 1: + loop_dims = ["K"] + case 2: + loop_dims = ["D", "K"] + case 3: + loop_dims = ["B", "D", "K"] + case 4: + loop_dims = ["B", "H", "D", "K"] + case _: + raise NotImplementedError + + loop_dims_broadcast = reversed( + [dim for dim, size in zip(reversed(loop_dims), reversed(broadcast_shape)) if size > 1] + ) + + equation_dims_common = "".join([f"[{dim.lower()}]" for dim in loop_dims]) + equation_dims_broadcast = "".join([f"[{dim.lower()}]" for dim in loop_dims_broadcast]) + equation = f"O{equation_dims_common}+=I{equation_dims_common}*W{equation_dims_broadcast}" + + data["loop_dims"] = loop_dims + data["equation"] = equation + + return data diff --git a/stream/parser/onnx/simd.py b/stream/parser/onnx/simd.py index 9dae37d3..32d83b1e 100644 --- a/stream/parser/onnx/simd.py +++ b/stream/parser/onnx/simd.py @@ -6,6 +6,7 @@ class SimdParser(OnnxComputeOperatorParser): """Parses an ONNX operator representing an elementwise operation (simd) into a ComputationNode. e.g. Add, etc. + # TODO this functionality is exactly the same as Mul but without support for broadcast (asymmetric) shapes """ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[int]): diff --git a/stream/stages/generation/tiled_workload_generation.py b/stream/stages/generation/tiled_workload_generation.py index 88479341..36560dad 100644 --- a/stream/stages/generation/tiled_workload_generation.py +++ b/stream/stages/generation/tiled_workload_generation.py @@ -395,6 +395,12 @@ def bounding_box_generator( inclusive_ranges = self.convert_to_inclusive_data_range(node.loop_ranges) dimensions = node.operand_dimensionality_order[operand] bounds = self.get_bounding_box_dimensions(producer, consumer, dimensions, inclusive_ranges) + + # TODO this is a whacky fix + # RTree doesn't accept bound of one dimension + if len(bounds) == 2: + bounds = (0, 0) + bounds + yield (i, bounds, None) def get_nb_input_dimensions(self, node: ComputationNode, operand: LayerOperand): @@ -416,7 +422,7 @@ def build_rtree( """ props = index.Property() # We assume all nodes in 'nodes' have identical dimensions - props.dimension = self.get_nb_input_dimensions(nodes[0], operand) + props.dimension = max(self.get_nb_input_dimensions(nodes[0], operand), 2) rtree = index.Index(self.bounding_box_generator(producer, consumer, nodes, operand), properties=props) return rtree diff --git a/stream/stages/generation/tiling_generation.py b/stream/stages/generation/tiling_generation.py index d0a463fb..70b3191d 100644 --- a/stream/stages/generation/tiling_generation.py +++ b/stream/stages/generation/tiling_generation.py @@ -184,7 +184,8 @@ def generate_inter_core_tiling(self, node: ComputationNode) -> TILING_T: if dim in node.layer_dim_sizes and node.layer_dim_sizes[dim] > 1: return [(dim, "*")] - raise ValueError("Unknown what loop dim to split across cores") + # No valid dim found -> just take someting + return [(next(iter(node.layer_dim_sizes)), "*")] @staticmethod def split_operator(model: ModelProto, node_name: str, num_splits: int): From 9fa69b4abbe4d21635589eafba19e6d5317da75f Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Thu, 7 Nov 2024 20:28:34 +0100 Subject: [PATCH 5/8] some parsing bugfixes --- iismodel.ilp | 123 +++++++++++++++++++++++++++++++++++++ stream/parser/onnx/conv.py | 13 ++-- stream/parser/onnx/mul.py | 33 +++++++++- 3 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 iismodel.ilp diff --git a/iismodel.ilp b/iismodel.ilp new file mode 100644 index 00000000..813ee161 --- /dev/null +++ b/iismodel.ilp @@ -0,0 +1,123 @@ +\ Model scheduling_copy +\ LP format - for model browsing. Use MPS format to capture full model detail. +Minimize + +Subject To + R26: core_assignments[Core_5,7000] <= 0 + R65: core_assignments[Core_5,7000] - assignments[Core_5,0,7000] + - assignments[Core_5,1,7000] - assignments[Core_5,2,7000] = 0 + R89: - node_assignments[1,7000] - 2 node_assignments[2,7000] + + slot_per_id[7000] = 0 + R91: - slot_per_id[5000] + slot_per_id[7000] >= 1 + R92: weights_per_core[Core_0] <= 1.6777216e+07 + R93: weights_per_core[Core_1] <= 1.6777216e+07 + R94: weights_per_core[Core_2] <= 1.6777216e+07 + R95: weights_per_core[Core_3] <= 1.6777216e+07 + R96: weights_per_core[Core_4] <= 1.048576e+06 + qc0: [ - k_splits[7000] * node_assignments[1,7000] + + node_assignments[1,7000] * assignments[Core_0,1,7000] + + node_assignments[1,7000] * assignments[Core_1,1,7000] + + node_assignments[1,7000] * assignments[Core_2,1,7000] + + node_assignments[1,7000] * assignments[Core_3,1,7000] + + node_assignments[1,7000] * assignments[Core_4,1,7000] + + node_assignments[1,7000] * assignments[Core_5,1,7000] ] = 0 + qc1: [ - k_splits[7000] * node_assignments[2,7000] + + node_assignments[2,7000] * assignments[Core_0,2,7000] + + node_assignments[2,7000] * assignments[Core_1,2,7000] + + node_assignments[2,7000] * assignments[Core_2,2,7000] + + node_assignments[2,7000] * assignments[Core_3,2,7000] + + node_assignments[2,7000] * assignments[Core_4,2,7000] + + node_assignments[2,7000] * assignments[Core_5,2,7000] ] = 0 + qc2: [ k_splits[7000] * weights_per_split[7000] ] >= 2.12992e+08 + qc3: weights_per_core[Core_0] + [ + - assignments[Core_0,0,2000] * weights_per_split[2000] + - assignments[Core_0,0,5000] * weights_per_split[5000] + - assignments[Core_0,0,7000] * weights_per_split[7000] + - assignments[Core_0,1,2000] * weights_per_split[2000] + - assignments[Core_0,1,5000] * weights_per_split[5000] + - assignments[Core_0,1,7000] * weights_per_split[7000] + - assignments[Core_0,2,2000] * weights_per_split[2000] + - assignments[Core_0,2,5000] * weights_per_split[5000] + - assignments[Core_0,2,7000] * weights_per_split[7000] ] >= 0 + qc4: weights_per_core[Core_1] + [ + - assignments[Core_1,0,2000] * weights_per_split[2000] + - assignments[Core_1,0,5000] * weights_per_split[5000] + - assignments[Core_1,0,7000] * weights_per_split[7000] + - assignments[Core_1,1,2000] * weights_per_split[2000] + - assignments[Core_1,1,5000] * weights_per_split[5000] + - assignments[Core_1,1,7000] * weights_per_split[7000] + - assignments[Core_1,2,2000] * weights_per_split[2000] + - assignments[Core_1,2,5000] * weights_per_split[5000] + - assignments[Core_1,2,7000] * weights_per_split[7000] ] >= 0 + qc5: weights_per_core[Core_2] + [ + - assignments[Core_2,0,2000] * weights_per_split[2000] + - assignments[Core_2,0,5000] * weights_per_split[5000] + - assignments[Core_2,0,7000] * weights_per_split[7000] + - assignments[Core_2,1,2000] * weights_per_split[2000] + - assignments[Core_2,1,5000] * weights_per_split[5000] + - assignments[Core_2,1,7000] * weights_per_split[7000] + - assignments[Core_2,2,2000] * weights_per_split[2000] + - assignments[Core_2,2,5000] * weights_per_split[5000] + - assignments[Core_2,2,7000] * weights_per_split[7000] ] >= 0 + qc6: weights_per_core[Core_3] + [ + - assignments[Core_3,0,2000] * weights_per_split[2000] + - assignments[Core_3,0,5000] * weights_per_split[5000] + - assignments[Core_3,0,7000] * weights_per_split[7000] + - assignments[Core_3,1,2000] * weights_per_split[2000] + - assignments[Core_3,1,5000] * weights_per_split[5000] + - assignments[Core_3,1,7000] * weights_per_split[7000] + - assignments[Core_3,2,2000] * weights_per_split[2000] + - assignments[Core_3,2,5000] * weights_per_split[5000] + - assignments[Core_3,2,7000] * weights_per_split[7000] ] >= 0 + qc7: weights_per_core[Core_4] + [ + - assignments[Core_4,0,2000] * weights_per_split[2000] + - assignments[Core_4,0,5000] * weights_per_split[5000] + - assignments[Core_4,0,7000] * weights_per_split[7000] + - assignments[Core_4,1,2000] * weights_per_split[2000] + - assignments[Core_4,1,5000] * weights_per_split[5000] + - assignments[Core_4,1,7000] * weights_per_split[7000] + - assignments[Core_4,2,2000] * weights_per_split[2000] + - assignments[Core_4,2,5000] * weights_per_split[5000] + - assignments[Core_4,2,7000] * weights_per_split[7000] ] >= 0 +Bounds + k_splits[7000] free + slot_per_id[7000] free + weights_per_split[7000] free + weights_per_core[Core_0] free + weights_per_core[Core_1] free + weights_per_core[Core_2] free + weights_per_core[Core_3] free + weights_per_core[Core_4] free +Binaries + core_assignments[Core_5,7000] node_assignments[1,7000] + node_assignments[2,7000] assignments[Core_0,0,2000] + assignments[Core_0,0,5000] assignments[Core_0,0,7000] + assignments[Core_0,1,2000] assignments[Core_0,1,5000] + assignments[Core_0,1,7000] assignments[Core_0,2,2000] + assignments[Core_0,2,5000] assignments[Core_0,2,7000] + assignments[Core_1,0,2000] assignments[Core_1,0,5000] + assignments[Core_1,0,7000] assignments[Core_1,1,2000] + assignments[Core_1,1,5000] assignments[Core_1,1,7000] + assignments[Core_1,2,2000] assignments[Core_1,2,5000] + assignments[Core_1,2,7000] assignments[Core_2,0,2000] + assignments[Core_2,0,5000] assignments[Core_2,0,7000] + assignments[Core_2,1,2000] assignments[Core_2,1,5000] + assignments[Core_2,1,7000] assignments[Core_2,2,2000] + assignments[Core_2,2,5000] assignments[Core_2,2,7000] + assignments[Core_3,0,2000] assignments[Core_3,0,5000] + assignments[Core_3,0,7000] assignments[Core_3,1,2000] + assignments[Core_3,1,5000] assignments[Core_3,1,7000] + assignments[Core_3,2,2000] assignments[Core_3,2,5000] + assignments[Core_3,2,7000] assignments[Core_4,0,2000] + assignments[Core_4,0,5000] assignments[Core_4,0,7000] + assignments[Core_4,1,2000] assignments[Core_4,1,5000] + assignments[Core_4,1,7000] assignments[Core_4,2,2000] + assignments[Core_4,2,5000] assignments[Core_4,2,7000] + assignments[Core_5,0,7000] assignments[Core_5,1,7000] + assignments[Core_5,2,7000] +Generals + k_splits[7000] slot_per_id[5000] slot_per_id[7000] weights_per_split[2000] + weights_per_split[5000] weights_per_split[7000] weights_per_core[Core_0] + weights_per_core[Core_1] weights_per_core[Core_2] weights_per_core[Core_3] + weights_per_core[Core_4] +End diff --git a/stream/parser/onnx/conv.py b/stream/parser/onnx/conv.py index af53bb26..d286e4c5 100644 --- a/stream/parser/onnx/conv.py +++ b/stream/parser/onnx/conv.py @@ -113,11 +113,14 @@ def get_layer_node_user_format( [padding[1], padding[3]], ] - # Remove dims with size 1 - dims_size_1 = [dim for dim, size in zip(data["loop_dims"], data["loop_sizes"]) if size == 1] - data["loop_sizes"] = [s for s in data["loop_sizes"] if s > 1] - data["loop_dims"] = [d for d in data["loop_dims"] if d not in dims_size_1] - for dim in dims_size_1: + # Remove dims with size 1, except batch + dim_sizes_larger_than_1 = { + dim: size for dim, size in zip(data["loop_dims"], data["loop_sizes"]) if size > 1 or dim == "B" + } + dims_with_size_1 = [dim for dim in data["loop_dims"] if dim not in dim_sizes_larger_than_1] + data["loop_dims"] = list(dim_sizes_larger_than_1.keys()) + data["loop_sizes"] = list(dim_sizes_larger_than_1.values()) + for dim in dims_with_size_1: data["equation"] = data["equation"].replace(f"[{dim.lower()}]", "") # Filter out loops with size 1 diff --git a/stream/parser/onnx/mul.py b/stream/parser/onnx/mul.py index 1d78fced..8cb43702 100644 --- a/stream/parser/onnx/mul.py +++ b/stream/parser/onnx/mul.py @@ -1,5 +1,7 @@ from typing import Any +from numpy import broadcast, broadcast_shapes + from stream.onnx_utils import get_onnx_input_shapes, get_onnx_output_shapes from stream.parser.onnx.operator_parser import OnnxComputeOperatorParser @@ -32,6 +34,35 @@ def get_common_and_broadcast_shape(self): return input_shape, broadcast_shape + def get_operand_source_input_format(self, shape_of_w: list[int]): + """This method needs more care in this subclass, since the equation assumes that the input with 'broadcast' + shape is always at `W`""" + predecessors = self.get_node_predecessors() + match len(predecessors): + case 1: + # One source operand, one constant + return {"W": self.node_id, "I": predecessors[0]} + case 2: + # Two source operands, none are constant + # Name of the input that corresponds to the W shape + broadcast_intput = self.node.input[get_onnx_input_shapes(self.node, self.onnx_model).index(shape_of_w)] + try: + node_id_W = next( + node_id + for node_id, outputs in self.nodes_outputs.items() + if broadcast_intput in outputs and node_id in predecessors + ) + node_id_I = ( + node_id_W + if predecessors[0] == predecessors[1] + else next(i for i in predecessors if i != node_id_W) + ) + return {"W": node_id_W, "I": node_id_I} + except StopIteration: + raise ValueError(f"Cannot find correct inputs of {self .node.name}") + case _: + raise ValueError("No more than 2 layer predecessors expected") + def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[int]): """ Generate the necessary dictionary items required for the LayerNode creation. @@ -42,7 +73,7 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ data["id"] = self.node_id data["name"] = self.node.name data["operator_type"] = self.node.op_type - data["operand_source"] = self.get_operand_source_input_format() + data["operand_source"] = self.get_operand_source_input_format(shape_of_w=broadcast_shape) data["operand_precision"] = self.get_operand_precision_user_format() data["dimension_relations"] = [] data["loop_sizes"] = common_shape From a38309a511832e75091a51e945dbb1eb1a2e29e0 Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Fri, 8 Nov 2024 10:09:04 +0100 Subject: [PATCH 6/8] fix bug in mulparser: don't remove dims of size 1 --- iismodel.ilp | 123 -------------------------------------- stream/parser/onnx/mul.py | 6 +- 2 files changed, 1 insertion(+), 128 deletions(-) delete mode 100644 iismodel.ilp diff --git a/iismodel.ilp b/iismodel.ilp deleted file mode 100644 index 813ee161..00000000 --- a/iismodel.ilp +++ /dev/null @@ -1,123 +0,0 @@ -\ Model scheduling_copy -\ LP format - for model browsing. Use MPS format to capture full model detail. -Minimize - -Subject To - R26: core_assignments[Core_5,7000] <= 0 - R65: core_assignments[Core_5,7000] - assignments[Core_5,0,7000] - - assignments[Core_5,1,7000] - assignments[Core_5,2,7000] = 0 - R89: - node_assignments[1,7000] - 2 node_assignments[2,7000] - + slot_per_id[7000] = 0 - R91: - slot_per_id[5000] + slot_per_id[7000] >= 1 - R92: weights_per_core[Core_0] <= 1.6777216e+07 - R93: weights_per_core[Core_1] <= 1.6777216e+07 - R94: weights_per_core[Core_2] <= 1.6777216e+07 - R95: weights_per_core[Core_3] <= 1.6777216e+07 - R96: weights_per_core[Core_4] <= 1.048576e+06 - qc0: [ - k_splits[7000] * node_assignments[1,7000] - + node_assignments[1,7000] * assignments[Core_0,1,7000] - + node_assignments[1,7000] * assignments[Core_1,1,7000] - + node_assignments[1,7000] * assignments[Core_2,1,7000] - + node_assignments[1,7000] * assignments[Core_3,1,7000] - + node_assignments[1,7000] * assignments[Core_4,1,7000] - + node_assignments[1,7000] * assignments[Core_5,1,7000] ] = 0 - qc1: [ - k_splits[7000] * node_assignments[2,7000] - + node_assignments[2,7000] * assignments[Core_0,2,7000] - + node_assignments[2,7000] * assignments[Core_1,2,7000] - + node_assignments[2,7000] * assignments[Core_2,2,7000] - + node_assignments[2,7000] * assignments[Core_3,2,7000] - + node_assignments[2,7000] * assignments[Core_4,2,7000] - + node_assignments[2,7000] * assignments[Core_5,2,7000] ] = 0 - qc2: [ k_splits[7000] * weights_per_split[7000] ] >= 2.12992e+08 - qc3: weights_per_core[Core_0] + [ - - assignments[Core_0,0,2000] * weights_per_split[2000] - - assignments[Core_0,0,5000] * weights_per_split[5000] - - assignments[Core_0,0,7000] * weights_per_split[7000] - - assignments[Core_0,1,2000] * weights_per_split[2000] - - assignments[Core_0,1,5000] * weights_per_split[5000] - - assignments[Core_0,1,7000] * weights_per_split[7000] - - assignments[Core_0,2,2000] * weights_per_split[2000] - - assignments[Core_0,2,5000] * weights_per_split[5000] - - assignments[Core_0,2,7000] * weights_per_split[7000] ] >= 0 - qc4: weights_per_core[Core_1] + [ - - assignments[Core_1,0,2000] * weights_per_split[2000] - - assignments[Core_1,0,5000] * weights_per_split[5000] - - assignments[Core_1,0,7000] * weights_per_split[7000] - - assignments[Core_1,1,2000] * weights_per_split[2000] - - assignments[Core_1,1,5000] * weights_per_split[5000] - - assignments[Core_1,1,7000] * weights_per_split[7000] - - assignments[Core_1,2,2000] * weights_per_split[2000] - - assignments[Core_1,2,5000] * weights_per_split[5000] - - assignments[Core_1,2,7000] * weights_per_split[7000] ] >= 0 - qc5: weights_per_core[Core_2] + [ - - assignments[Core_2,0,2000] * weights_per_split[2000] - - assignments[Core_2,0,5000] * weights_per_split[5000] - - assignments[Core_2,0,7000] * weights_per_split[7000] - - assignments[Core_2,1,2000] * weights_per_split[2000] - - assignments[Core_2,1,5000] * weights_per_split[5000] - - assignments[Core_2,1,7000] * weights_per_split[7000] - - assignments[Core_2,2,2000] * weights_per_split[2000] - - assignments[Core_2,2,5000] * weights_per_split[5000] - - assignments[Core_2,2,7000] * weights_per_split[7000] ] >= 0 - qc6: weights_per_core[Core_3] + [ - - assignments[Core_3,0,2000] * weights_per_split[2000] - - assignments[Core_3,0,5000] * weights_per_split[5000] - - assignments[Core_3,0,7000] * weights_per_split[7000] - - assignments[Core_3,1,2000] * weights_per_split[2000] - - assignments[Core_3,1,5000] * weights_per_split[5000] - - assignments[Core_3,1,7000] * weights_per_split[7000] - - assignments[Core_3,2,2000] * weights_per_split[2000] - - assignments[Core_3,2,5000] * weights_per_split[5000] - - assignments[Core_3,2,7000] * weights_per_split[7000] ] >= 0 - qc7: weights_per_core[Core_4] + [ - - assignments[Core_4,0,2000] * weights_per_split[2000] - - assignments[Core_4,0,5000] * weights_per_split[5000] - - assignments[Core_4,0,7000] * weights_per_split[7000] - - assignments[Core_4,1,2000] * weights_per_split[2000] - - assignments[Core_4,1,5000] * weights_per_split[5000] - - assignments[Core_4,1,7000] * weights_per_split[7000] - - assignments[Core_4,2,2000] * weights_per_split[2000] - - assignments[Core_4,2,5000] * weights_per_split[5000] - - assignments[Core_4,2,7000] * weights_per_split[7000] ] >= 0 -Bounds - k_splits[7000] free - slot_per_id[7000] free - weights_per_split[7000] free - weights_per_core[Core_0] free - weights_per_core[Core_1] free - weights_per_core[Core_2] free - weights_per_core[Core_3] free - weights_per_core[Core_4] free -Binaries - core_assignments[Core_5,7000] node_assignments[1,7000] - node_assignments[2,7000] assignments[Core_0,0,2000] - assignments[Core_0,0,5000] assignments[Core_0,0,7000] - assignments[Core_0,1,2000] assignments[Core_0,1,5000] - assignments[Core_0,1,7000] assignments[Core_0,2,2000] - assignments[Core_0,2,5000] assignments[Core_0,2,7000] - assignments[Core_1,0,2000] assignments[Core_1,0,5000] - assignments[Core_1,0,7000] assignments[Core_1,1,2000] - assignments[Core_1,1,5000] assignments[Core_1,1,7000] - assignments[Core_1,2,2000] assignments[Core_1,2,5000] - assignments[Core_1,2,7000] assignments[Core_2,0,2000] - assignments[Core_2,0,5000] assignments[Core_2,0,7000] - assignments[Core_2,1,2000] assignments[Core_2,1,5000] - assignments[Core_2,1,7000] assignments[Core_2,2,2000] - assignments[Core_2,2,5000] assignments[Core_2,2,7000] - assignments[Core_3,0,2000] assignments[Core_3,0,5000] - assignments[Core_3,0,7000] assignments[Core_3,1,2000] - assignments[Core_3,1,5000] assignments[Core_3,1,7000] - assignments[Core_3,2,2000] assignments[Core_3,2,5000] - assignments[Core_3,2,7000] assignments[Core_4,0,2000] - assignments[Core_4,0,5000] assignments[Core_4,0,7000] - assignments[Core_4,1,2000] assignments[Core_4,1,5000] - assignments[Core_4,1,7000] assignments[Core_4,2,2000] - assignments[Core_4,2,5000] assignments[Core_4,2,7000] - assignments[Core_5,0,7000] assignments[Core_5,1,7000] - assignments[Core_5,2,7000] -Generals - k_splits[7000] slot_per_id[5000] slot_per_id[7000] weights_per_split[2000] - weights_per_split[5000] weights_per_split[7000] weights_per_core[Core_0] - weights_per_core[Core_1] weights_per_core[Core_2] weights_per_core[Core_3] - weights_per_core[Core_4] -End diff --git a/stream/parser/onnx/mul.py b/stream/parser/onnx/mul.py index 8cb43702..303d0769 100644 --- a/stream/parser/onnx/mul.py +++ b/stream/parser/onnx/mul.py @@ -1,7 +1,5 @@ from typing import Any -from numpy import broadcast, broadcast_shapes - from stream.onnx_utils import get_onnx_input_shapes, get_onnx_output_shapes from stream.parser.onnx.operator_parser import OnnxComputeOperatorParser @@ -90,9 +88,7 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ case _: raise NotImplementedError - loop_dims_broadcast = reversed( - [dim for dim, size in zip(reversed(loop_dims), reversed(broadcast_shape)) if size > 1] - ) + loop_dims_broadcast = reversed([dim for dim, _ in zip(reversed(loop_dims), reversed(broadcast_shape))]) equation_dims_common = "".join([f"[{dim.lower()}]" for dim in loop_dims]) equation_dims_broadcast = "".join([f"[{dim.lower()}]" for dim in loop_dims_broadcast]) From d1db0d9133bc1b5f960d5a21ef78573f8151ef4a Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Fri, 8 Nov 2024 10:51:00 +0100 Subject: [PATCH 7/8] fix bug in conv: dont remove dims of size 1, except K and C (not present in some 1D convs) --- stream/parser/onnx/conv.py | 31 ++++++++------------- stream/parser/onnx/operator_parser.py | 8 ++++++ stream/workload/computation/pooling_node.py | 4 +++ 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/stream/parser/onnx/conv.py b/stream/parser/onnx/conv.py index d286e4c5..d22dafc0 100644 --- a/stream/parser/onnx/conv.py +++ b/stream/parser/onnx/conv.py @@ -83,8 +83,7 @@ def get_layer_node_user_format( if is_1d_conv: # No FY, OY, IY - data["loop_sizes"] = [B, K, G, OX, C, FX] - data["loop_dims"] = ["B", "K", "G", "OX", "C", "FX"] + loop_size_dict = {"B": B, "K": K, "G": G, "OX": OX, "C": C, "FX": FX} data["equation"] = f"O[b][g][k][ox]+=W[{weight_dim}][c][fx]*I[b][g][c][ix]" data["pr_loop_dims"] = ["IX"] data["pr_loop_sizes"] = [IX] @@ -99,8 +98,7 @@ def get_layer_node_user_format( FY = kernel_shape[1] # TODO is kernel_shape in (FX, FY) format or (FY, FX)? (I assumed the former) IY = input_shape[3] OY = output_shape[3] - data["loop_sizes"] = [B, K, G, OX, C, FX, OY, FY] - data["loop_dims"] = ["B", "K", "G", "OX", "C", "FX", "OY", "FY"] + loop_size_dict = {"B": B, "K": K, "G": G, "OX": OX, "C": C, "FX": FX, "OY": OY, "FY": FY} data["equation"] = f"O[b][g][k][oy][ox]+=W[{weight_dim}][c][fy][fx]*I[b][g][c][iy][ix]" data["pr_loop_dims"] = ["IX", "IY"] data["pr_loop_sizes"] = [IX, IY] @@ -113,22 +111,15 @@ def get_layer_node_user_format( [padding[1], padding[3]], ] - # Remove dims with size 1, except batch - dim_sizes_larger_than_1 = { - dim: size for dim, size in zip(data["loop_dims"], data["loop_sizes"]) if size > 1 or dim == "B" - } - dims_with_size_1 = [dim for dim in data["loop_dims"] if dim not in dim_sizes_larger_than_1] - data["loop_dims"] = list(dim_sizes_larger_than_1.keys()) - data["loop_sizes"] = list(dim_sizes_larger_than_1.values()) - for dim in dims_with_size_1: - data["equation"] = data["equation"].replace(f"[{dim.lower()}]", "") - - # Filter out loops with size 1 - # loop_sizes = {"B": B, "K": K, "G": G, "OX": OX, "OY": OY, "C": C, "FX": FX, "FY": FY} - # dims_with_size_1 = [k for k, v in loop_sizes.items() if v == 1] - # loop_sizes = {k: v for k, v in loop_sizes.items() if v > 1} - # data["loop_dims"] = list(loop_sizes.keys()) - # data["loop_sizes"] = list(loop_sizes.values()) + # Remove C/K if they have size 1 + for dim in ["C", "K"]: + if loop_size_dict[dim] == 1: + del loop_size_dict[dim] + # Remove from equation + data["equation"] = data["equation"].replace(f"[{dim.lower()}]", "") + + data["loop_dims"] = list(loop_size_dict.keys()) + data["loop_sizes"] = list(loop_size_dict.values()) return data diff --git a/stream/parser/onnx/operator_parser.py b/stream/parser/onnx/operator_parser.py index a4345895..78d5e99c 100644 --- a/stream/parser/onnx/operator_parser.py +++ b/stream/parser/onnx/operator_parser.py @@ -68,6 +68,14 @@ def get_operand_precision_user_format(self) -> dict[str, int]: intermediate_output_precision: int = self.get_intermediate_output_precision() predecessors = self.get_node_predecessors() match len(predecessors): + case 0: + # e.g. the first node in the network -> assume only one variable input + return { + "W": weight_precision, + "I": act_precision, + "O_final": act_precision, + "O": intermediate_output_precision, + } case 1: # One source operand, one constant return { diff --git a/stream/workload/computation/pooling_node.py b/stream/workload/computation/pooling_node.py index 0c4151c5..74a27169 100644 --- a/stream/workload/computation/pooling_node.py +++ b/stream/workload/computation/pooling_node.py @@ -5,12 +5,15 @@ class PoolingNode(ComputationNode): + """TODO this node can be replaced by instantiating ComputationNode directly""" + def __init__( self, node_id: int, node_name: str, node_attr: LayerNodeAttributes, mapping_attr: InterCoreMappingAttributes, + input_names: list[str] = [], ): super().__init__( node_id=node_id, @@ -18,4 +21,5 @@ def __init__( node_attr=node_attr, mapping_attr=mapping_attr, op_type="pooling", + input_names=input_names, ) From 5485268a95b72be477c1e719a5fed2f42074ebaf Mon Sep 17 00:00:00 2001 From: RobinGeens Date: Fri, 8 Nov 2024 17:41:20 +0100 Subject: [PATCH 8/8] bugfix in reduce_1d: explicitly manage the keep_dim option --- stream/parser/onnx/conv.py | 12 ------ stream/parser/onnx/model.py | 9 +++- stream/parser/onnx/mul.py | 3 ++ stream/parser/onnx/operator_parser.py | 3 ++ stream/parser/onnx/reduce_1d.py | 42 ++++++++++++++++--- .../generation/tiled_workload_generation.py | 1 + 6 files changed, 52 insertions(+), 18 deletions(-) diff --git a/stream/parser/onnx/conv.py b/stream/parser/onnx/conv.py index d22dafc0..1181c176 100644 --- a/stream/parser/onnx/conv.py +++ b/stream/parser/onnx/conv.py @@ -49,18 +49,6 @@ def get_layer_node_user_format( # 1D Conv case: append dimensions of size 1 so equation holds. Conv in FY dimension is_1d_conv = len(kernel_shape) == 1 - # if len(kernel_shape) == 1: - # kernel_shape.insert(0, 1) - # input_shape.append(1) - # output_shape.append(1) - # strides.append(1) - # dilations.append(1) - # assert len(input_shape) == 4 - # assert len(output_shape) == 4 - - # if len(padding) == 2: - # padding = 2 * padding - # Get dimension sizes from input parameters assert input_shape[0] == output_shape[0], "Batch size is different for input and output activations." B = output_shape[0] diff --git a/stream/parser/onnx/model.py b/stream/parser/onnx/model.py index 7ec7f96a..7109471a 100644 --- a/stream/parser/onnx/model.py +++ b/stream/parser/onnx/model.py @@ -17,6 +17,7 @@ from stream.parser.onnx.mul import MulParser from stream.parser.onnx.operator_parser import OnnxOperatorParser from stream.parser.onnx.pooling import PoolingParser +from stream.parser.onnx.reduce_1d import Reduce1DParser from stream.parser.onnx.reshape import ReshapeParser from stream.parser.onnx.simd import SimdParser from stream.parser.onnx.slice import SliceParser @@ -34,6 +35,7 @@ class ONNXModelParser: # Map the node's op_type to the corresponding Parser class OP_TYPE_TO_PARSER: dict[str, Type[OnnxOperatorParser]] = { + # General "QLinearConv": ConvParser, "Conv": ConvParser, "MatMul": MatMulParser, @@ -46,10 +48,15 @@ class ONNXModelParser: "Add": MulParser, "Mul": MulParser, "Softmax": SoftmaxParser, - # Activations + # Single-input element-wise + "ReduceMean": Reduce1DParser, "Relu": SimdParser, "Gelu": SimdParser, "Silu": SimdParser, + "Sqrt": SimdParser, + "Div": SimdParser, + "Pow": SimdParser, + "Reciprocal": SimdParser, # Div with 1 as numerator # Dependency propagation "LpNormalization": LpNormalizationParser, "Gather": GatherParser, diff --git a/stream/parser/onnx/mul.py b/stream/parser/onnx/mul.py index 303d0769..612a9406 100644 --- a/stream/parser/onnx/mul.py +++ b/stream/parser/onnx/mul.py @@ -37,6 +37,9 @@ def get_operand_source_input_format(self, shape_of_w: list[int]): shape is always at `W`""" predecessors = self.get_node_predecessors() match len(predecessors): + case 0: + # e.g. first node of graph + return {"W": self.node_id, "I": self.node_id} case 1: # One source operand, one constant return {"W": self.node_id, "I": predecessors[0]} diff --git a/stream/parser/onnx/operator_parser.py b/stream/parser/onnx/operator_parser.py index 78d5e99c..e288d18d 100644 --- a/stream/parser/onnx/operator_parser.py +++ b/stream/parser/onnx/operator_parser.py @@ -40,6 +40,9 @@ def generate_node(self) -> Node: ... def get_operand_source_input_format(self): predecessors = self.get_node_predecessors() match len(predecessors): + case 0: + # e.g. first node of graph + return {"W": self.node_id, "I": self.node_id} case 1: # One source operand, one constant return {"W": self.node_id, "I": predecessors[0]} diff --git a/stream/parser/onnx/reduce_1d.py b/stream/parser/onnx/reduce_1d.py index b34289b0..26f8d7ff 100644 --- a/stream/parser/onnx/reduce_1d.py +++ b/stream/parser/onnx/reduce_1d.py @@ -8,12 +8,36 @@ class Reduce1DParser(OnnxComputeOperatorParser): e.g. sum over one row or max of a single row """ + def get_reduction_dim(self, input_shape: list[int], output_shape: list[int]): + """Returns the axis in which the dimension is reduced""" + + # The case that keepdim=True: the reduced dimension is kept with size 1 + if len(input_shape) == len(output_shape): + different_size = [a != b for a, b in zip(input_shape, output_shape)] + if sum(different_size) != 1: + raise ValueError(f"Input and output shapes {input_shape}, {output_shape} should only differ in one dim") + reduction_dim = different_size.index(True) + if output_shape[reduction_dim] != 1: + raise ValueError(f"The reduced dimension at axis {reduction_dim} in {output_shape} is larger than 1") + return reduction_dim + + # Other: assume that the reduction is at axis=-1 + if not all(a == b for a, b in zip(input_shape, output_shape)): + raise NotImplementedError("Reduce node with reduction axis other than -1 not implemented yet.") + reduction_dim = len(input_shape) - 1 # Last dimension + def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[int]): """ Generate the necessary dictionary items required for the LayerNode creation. """ - # TODO check the output shape as well? - assert len(self.get_node_predecessors()) == 1 + if len(self.get_node_predecessors()) != 1: + raise NotImplementedError + + if self.get_reduction_dim(input_shape, output_shape) != len(input_shape) - 1: + raise NotImplementedError("Only reduction in axis=-1 is supported") + + # This is a ONNX node property but can be inferred from the shapes + keep_dim = len(input_shape) == len(output_shape) data: dict[str, Any] = {} data["id"] = self.node_id @@ -24,17 +48,25 @@ def get_layer_node_user_format(self, input_shape: list[int], output_shape: list[ data["dimension_relations"] = [] data["loop_sizes"] = input_shape + # C is always the reduction dim + # If keep_dim: add an arbitrary dim of size 1 + reduced_dim_output = "CR" # C reduced to 1 + eq_part_CR = f"[{reduced_dim_output}]" if keep_dim else "" match len(input_shape): case 2: - data["equation"] = "O[k]+=I[k][c]*W[]" + data["equation"] = f"O[k]{eq_part_CR}+=I[k][c]*W[]" data["loop_dims"] = ["K", "C"] case 3: - data["equation"] = "O[b][k]+=I[b][k][c]*W[]" + data["equation"] = f"O[b][k]{eq_part_CR}+=I[b][k][c]*W[]" data["loop_dims"] = ["B", "K", "C"] case 4: - data["equation"] = "O[b][h][k]+=I[b][h][k][c]*W[]" + data["equation"] = f"O[b][h][k]{eq_part_CR}+=I[b][h][k][c]*W[]" data["loop_dims"] = ["B", "H", "K", "C"] case _: raise NotImplementedError + if keep_dim: + data["loop_dims"] += [reduced_dim_output] + data["loop_sizes"] += [1] + return data diff --git a/stream/stages/generation/tiled_workload_generation.py b/stream/stages/generation/tiled_workload_generation.py index 36560dad..5bb772f3 100644 --- a/stream/stages/generation/tiled_workload_generation.py +++ b/stream/stages/generation/tiled_workload_generation.py @@ -376,6 +376,7 @@ def get_bounding_box_dimensions( # where the onnx tensors are always flattened back to 4D (merging the G+C or G+K into one channel dimension) dimensions, loop_ranges = self.flatten_grouped_convolution_ranges(producer, consumer, dimensions, loop_ranges) bounding_box = [loop_ranges[dim] for dim in dimensions] + # TODO can bounding box have size 1? Will probably crash if so if not interleaved: bounding_box_flat = tuple([item for sublist in bounding_box for item in sublist])