diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml index c452b126fd..625de85fa1 100644 --- a/.github/workflows/buildAndTest.yml +++ b/.github/workflows/buildAndTest.yml @@ -52,6 +52,7 @@ jobs: - name: Install Python packages run: | pip install cmake numpy psutil pybind11 rich pkginfo lit PyYAML + pip install -r python/requirements.txt - name: Install packages run: sudo apt-get install -y ninja-build clang lld diff --git a/.github/workflows/lintAndFormat.yml b/.github/workflows/lintAndFormat.yml index fc437b12d8..e1dd9b5b23 100644 --- a/.github/workflows/lintAndFormat.yml +++ b/.github/workflows/lintAndFormat.yml @@ -40,6 +40,7 @@ jobs: - name: Install Python packages run: | pip install cmake numpy psutil pybind11 rich pkginfo lit PyYAML requests + pip install -r python/requirements.txt - name: Get MLIR id: mlir-wheels @@ -270,6 +271,7 @@ jobs: - name: Install Python and other packages run: | pip install cmake numpy psutil pybind11 rich lit + pip install -r python/requirements.txt - name: Install Ninja run: sudo apt-get install -y ninja-build clang lld llvm diff --git a/.pylintrc b/.pylintrc index c4b1380cf1..9e32b33e32 100644 --- a/.pylintrc +++ b/.pylintrc @@ -56,4 +56,7 @@ check-str-concat-over-line-jumps=yes # C0114: Missing module docstring (missing-module-docstring) # E0402: Attempted relative import beyond top-level package (relative-beyond-top-level) # C0115: Missing class docstring (missing-class-docstring) -disable=C0116,C0114,E0402,C0115 \ No newline at end of file +# C0415: Import outside toplevel +# R0913: Too many arguments +# R0914: Too many local variables +disable=C0116,C0114,E0402,C0115,C0415,R0913,R0914 \ No newline at end of file diff --git a/python/requirements.txt b/python/requirements.txt index 2dd6e968b2..2a990cfdc4 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -4,4 +4,6 @@ PyYAML>=5.3.1, <=6.0.1 dataclasses>=0.6, <=0.8 networkx[default] ortools -gurobipy \ No newline at end of file +gurobipy +matplotlib +ortools \ No newline at end of file diff --git a/python/util.py b/python/util.py index 6e0059c49e..32f3607b69 100644 --- a/python/util.py +++ b/python/util.py @@ -61,7 +61,10 @@ def infer_mlir_type( """ if isinstance(py_val, bool): return T.bool() - elif isinstance(py_val, int): + + if isinstance(py_val, int): + # no clue why but black can't decide which it wants the ** + # fmt: off if -(2 ** 31) <= py_val < 2 ** 31: return T.i32() elif 2 ** 31 <= py_val < 2 ** 32: @@ -70,9 +73,10 @@ def infer_mlir_type( return T.i64() elif 2 ** 63 <= py_val < 2 ** 64: return T.ui64() - else: - raise RuntimeError(f"Nonrepresentable integer {py_val}.") - elif isinstance(py_val, float): + raise RuntimeError(f"Nonrepresentable integer {py_val}.") + # fmt: on + + if isinstance(py_val, float): if ( abs(py_val) == float("inf") or abs(py_val) == 0.0 @@ -80,15 +84,15 @@ def infer_mlir_type( or np.finfo(np.float32).min <= abs(py_val) <= np.finfo(np.float32).max ): return T.f32() - else: - return T.f64() - elif isinstance(py_val, np.ndarray): + return T.f64() + + if isinstance(py_val, np.ndarray): dtype = np_dtype_to_mlir_type(py_val.dtype.type) return RankedTensorType.get(py_val.shape, dtype) - else: - raise NotImplementedError( - f"Unsupported Python value {py_val=} with type {type(py_val)}" - ) + + raise NotImplementedError( + f"Unsupported Python value {py_val=} with type {type(py_val)}" + ) def mlir_type_to_np_dtype(mlir_type): @@ -218,17 +222,17 @@ def route_using_cp( # what goes in must come out model.Add( - sum([flow_var[e] for e in DG.in_edges(nbunch=n)]) - == sum([flow_var[e] for e in DG.out_edges(nbunch=n)]) + sum(flow_var[e] for e in DG.in_edges(nbunch=n)) + == sum(flow_var[e] for e in DG.out_edges(nbunch=n)) ) # flow must leave src, and must not enter src - model.Add(sum([flow_var[src, j] for j in DG.neighbors(src)]) == 1) - model.Add(sum([flow_var[i, src] for i in DG.neighbors(src)]) == 0) + model.Add(sum(flow_var[src, j] for j in DG.neighbors(src)) == 1) + model.Add(sum(flow_var[i, src] for i in DG.neighbors(src)) == 0) # flow must enter tgt, and must not leave tgt - model.Add(sum([flow_var[tgt, j] for j in DG.neighbors(tgt)]) == 0) - model.Add(sum([flow_var[i, tgt] for i in DG.neighbors(tgt)]) == 1) + model.Add(sum(flow_var[tgt, j] for j in DG.neighbors(tgt)) == 0) + model.Add(sum(flow_var[i, tgt] for i in DG.neighbors(tgt)) == 1) # Create demand variables total_demand = { @@ -241,7 +245,7 @@ def route_using_cp( # Add demand/flow relationship for i, j, attrs in DG.edges(data=True): - model.Add(total_demand[i, j] == sum([f[i, j] for f in flat_flow_vars])) + model.Add(total_demand[i, j] == sum(f[i, j] for f in flat_flow_vars)) model.Add(total_demand[i, j] <= attrs["capacity"]) if min_edges: @@ -268,15 +272,15 @@ def route_using_cp( ) ) - obj = sum([total_demand[i, j] for i, j in DG.edges]) + obj = sum(total_demand[i, j] for i, j in DG.edges) if min_edges: - obj += sum([used_edges[i, j] for i, j in DG.edges]) + obj += sum(used_edges[i, j] for i, j in DG.edges) else: - obj += sum([overlapping_demands[i, j] for i, j in DG.edges]) + obj += sum(overlapping_demands[i, j] for i, j in DG.edges) model.Minimize(obj) status = solver.Solve(model) - if status == cp_model.OPTIMAL or status == cp_model.FEASIBLE: + if status in {cp_model.OPTIMAL, cp_model.FEASIBLE}: flow_paths = {} for flow, flow_varss in flow_vars.items(): flow_paths[flow] = [ diff --git a/test/python/python_passes.py b/test/python/python_passes.py index 31026304bc..95300002e6 100644 --- a/test/python/python_passes.py +++ b/test/python/python_passes.py @@ -1,19 +1,17 @@ # Copyright (C) 2022, Advanced Micro Devices, Inc. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -import os -from pathlib import Path # RUN: %PYTHON %s | FileCheck %s # REQUIRES: python_passes - +from pathlib import Path from pprint import pprint from aie._mlir_libs._aie_python_passes import ( create_python_router_pass, pass_manager_add_owned_pass, ) -from aie.dialects.aie import register_dialect + from aie.ir import Context, Location, Module from aie.passmanager import PassManager from aie.util import Router @@ -22,7 +20,7 @@ def run(f): with Context() as ctx, Location.unknown(): print("\nTEST:", f.__name__) - f(ctx) + f() THIS_FILE = __file__ @@ -30,10 +28,9 @@ def run(f): # CHECK-LABEL: TEST: test_broadcast @run -def test_broadcast(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "broadcast.mlir").read() - ) +def test_broadcast(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "broadcast.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -216,10 +213,9 @@ def test_broadcast(ctx): # CHECK-LABEL: TEST: test_flow_test_1 @run -def test_flow_test_1(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "flow_test_1.mlir").read() - ) +def test_flow_test_1(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "flow_test_1.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -236,10 +232,9 @@ def test_flow_test_1(ctx): # CHECK-LABEL: TEST: test_flow_test_2 @run -def test_flow_test_2(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "flow_test_2.mlir").read() - ) +def test_flow_test_2(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "flow_test_2.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -256,10 +251,9 @@ def test_flow_test_2(ctx): # CHECK-LABEL: TEST: test_flow_test_3 @run -def test_flow_test_3(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "flow_test_3.mlir").read() - ) +def test_flow_test_3(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "flow_test_3.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -276,10 +270,9 @@ def test_flow_test_3(ctx): # CHECK-LABEL: TEST: test_many_flows @run -def test_many_flows(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "many_flows.mlir").read() - ) +def test_many_flows(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "many_flows.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -296,10 +289,9 @@ def test_many_flows(ctx): # CHECK-LABEL: TEST: test_many_flows2 @run -def test_many_flows2(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "many_flows2.mlir").read() - ) +def test_many_flows2(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "many_flows2.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -316,10 +308,9 @@ def test_many_flows2(ctx): # CHECK-LABEL: TEST: test_memtile @run -def test_memtile(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "memtile.mlir").read() - ) +def test_memtile(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "memtile.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -336,14 +327,13 @@ def test_memtile(ctx): # CHECK-LABEL: TEST: test_memtile_routing_constraints @run -def test_memtile_routing_constraints(ctx): - mlir_module = Module.parse( - open( - Path(THIS_FILE).parent.parent - / "create-flows" - / "memtile_routing_constraints.mlir" - ).read() - ) +def test_memtile_routing_constraints(): + with open( + Path(THIS_FILE).parent.parent + / "create-flows" + / "memtile_routing_constraints.mlir" + ) as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -360,10 +350,9 @@ def test_memtile_routing_constraints(ctx): # CHECK-LABEL: TEST: test_mmult # @run -def test_mmult(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "mmult.mlir").read() - ) +def test_mmult(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "mmult.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -380,33 +369,31 @@ def test_mmult(ctx): # CHECK-LABEL: TEST: test_more_flows_shim @run -def test_more_flows_shim(ctx): - for mlir_module in ( - open(Path(THIS_FILE).parent.parent / "create-flows" / "more_flows_shim.mlir") - .read() - .split("// -----") - ): - mlir_module = Module.parse(mlir_module) - r = Router() - pass_ = create_python_router_pass(r) - pm = PassManager() - pass_manager_add_owned_pass(pm, pass_) - device = mlir_module.body.operations[0] - pm.run(device.operation) - - for src, paths in r.routing_solution.items(): - print(src) - pprint(paths) - - print(mlir_module) +def test_more_flows_shim(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "more_flows_shim.mlir" + ) as f: + for mlir_module in f.read().split("// -----"): + mlir_module = Module.parse(mlir_module) + r = Router() + pass_ = create_python_router_pass(r) + pm = PassManager() + pass_manager_add_owned_pass(pm, pass_) + device = mlir_module.body.operations[0] + pm.run(device.operation) + + for src, paths in r.routing_solution.items(): + print(src) + pprint(paths) + + print(mlir_module) # CHECK-LABEL: TEST: test_over_flows @run -def test_over_flows(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "over_flows.mlir").read() - ) +def test_over_flows(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "over_flows.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -423,12 +410,11 @@ def test_over_flows(ctx): # CHECK-LABEL: TEST: test_routed_herd_3x1 # @run -def test_routed_herd_3x1(ctx): - mlir_module = Module.parse( - open( - Path(THIS_FILE).parent.parent / "create-flows" / "routed_herd_3x1.mlir" - ).read() - ) +def test_routed_herd_3x1(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "routed_herd_3x1.mlir" + ) as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -445,12 +431,11 @@ def test_routed_herd_3x1(ctx): # CHECK-LABEL: TEST: test_routed_herd_3x2 # @run -def test_routed_herd_3x2(ctx): - mlir_module = Module.parse( - open( - Path(THIS_FILE).parent.parent / "create-flows" / "routed_herd_3x2.mlir" - ).read() - ) +def test_routed_herd_3x2(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "routed_herd_3x2.mlir" + ) as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -467,10 +452,9 @@ def test_routed_herd_3x2(ctx): # CHECK-LABEL: TEST: test_simple @run -def test_simple(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "simple.mlir").read() - ) +def test_simple(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "simple.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -505,10 +489,9 @@ def test_simple(ctx): # CHECK-LABEL: TEST: test_simple2 @run -def test_simple2(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "simple2.mlir").read() - ) +def test_simple2(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "simple2.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -538,12 +521,11 @@ def test_simple2(ctx): # CHECK-LABEL: TEST: test_simple_flows # @run -def test_simple_flows(ctx): - mlir_module = Module.parse( - open( - Path(THIS_FILE).parent.parent / "create-flows" / "simple_flows.mlir" - ).read() - ) +def test_simple_flows(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "simple_flows.mlir" + ) as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -560,12 +542,11 @@ def test_simple_flows(ctx): # CHECK-LABEL: TEST: test_simple_flows2 @run -def test_simple_flows2(ctx): - mlir_module = Module.parse( - open( - Path(THIS_FILE).parent.parent / "create-flows" / "simple_flows2.mlir" - ).read() - ) +def test_simple_flows2(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "simple_flows2.mlir" + ) as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -603,32 +584,30 @@ def test_simple_flows2(ctx): # CHECK-LABEL: TEST: test_simple_flows_shim @run -def test_simple_flows_shim(ctx): - for mlir_module in ( - open(Path(THIS_FILE).parent.parent / "create-flows" / "simple_flows_shim.mlir") - .read() - .split("// -----") +def test_simple_flows_shim(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "simple_flows_shim.mlir" ): - mlir_module = Module.parse(mlir_module) - r = Router() - pass_ = create_python_router_pass(r) - pm = PassManager() - pass_manager_add_owned_pass(pm, pass_) - device = mlir_module.body.operations[0] - pm.run(device.operation) + for mlir_module in f.read().split("// -----"): + mlir_module = Module.parse(mlir_module) + r = Router() + pass_ = create_python_router_pass(r) + pm = PassManager() + pass_manager_add_owned_pass(pm, pass_) + device = mlir_module.body.operations[0] + pm.run(device.operation) - for src, paths in r.routing_solution.items(): - print(src) - pprint(paths) + for src, paths in r.routing_solution.items(): + print(src) + pprint(paths) - print(mlir_module) + print(mlir_module) # @run -def test_vecmul_4x4(ctx): - mlir_module = Module.parse( - open(Path(THIS_FILE).parent.parent / "create-flows" / "vecmul_4x4.mlir").read() - ) +def test_vecmul_4x4(): + with open(Path(THIS_FILE).parent.parent / "create-flows" / "vecmul_4x4.mlir") as f: + mlir_module = Module.parse(f.read()) r = Router() pass_ = create_python_router_pass(r) pm = PassManager() @@ -644,52 +623,21 @@ def test_vecmul_4x4(ctx): # @run -def test_unit_fixed_connections(ctx): - for mlir_module in ( - open( - Path(THIS_FILE).parent.parent - / "create-flows" - / "unit_fixed_connections.mlir" - ) - .read() - .split("// -----") - ): - mlir_module = Module.parse(mlir_module) - r = Router() - pass_ = create_python_router_pass(r) - pm = PassManager() - pass_manager_add_owned_pass(pm, pass_) - device = mlir_module.body.operations[0] - pm.run(device.operation) - - for src, paths in r.routing_solution.items(): - print(src) - pprint(paths) - - print(mlir_module) - - -# @run -def test_unit_fixed_connections(ctx): - for mlir_module in ( - open( - Path(THIS_FILE).parent.parent - / "create-flows" - / "unit_fixed_connections.mlir" - ) - .read() - .split("// -----") - ): - mlir_module = Module.parse(mlir_module) - r = Router() - pass_ = create_python_router_pass(r) - pm = PassManager() - pass_manager_add_owned_pass(pm, pass_) - device = mlir_module.body.operations[0] - pm.run(device.operation) - - for src, paths in r.routing_solution.items(): - print(src) - pprint(paths) - - print(mlir_module) +def test_unit_fixed_connections(): + with open( + Path(THIS_FILE).parent.parent / "create-flows" / "unit_fixed_connections.mlir" + ) as f: + for mlir_module in f.read().split("// -----"): + mlir_module = Module.parse(mlir_module) + r = Router() + pass_ = create_python_router_pass(r) + pm = PassManager() + pass_manager_add_owned_pass(pm, pass_) + device = mlir_module.body.operations[0] + pm.run(device.operation) + + for src, paths in r.routing_solution.items(): + print(src) + pprint(paths) + + print(mlir_module)