Move aiecc.py implementation to python library (#387)

.github/workflows/buildAndTest.yml

@@ -179,7 +179,6 @@ jobs:
             -DCMAKE_LINKER=lld \
             -DCMAKE_C_COMPILER=clang \
             -DCMAKE_CXX_COMPILER=clang++ \
-            -DAIE_ENABLE_BINDINGS_PYTHON=ON \
             -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
             -DLLVM_EXTERNAL_LIT=`pwd`/../llvm/build/bin/llvm-lit \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
@@ -206,7 +205,6 @@ jobs:
             -DCMAKE_LINKER=lld \
             -DCMAKE_C_COMPILER=clang \
             -DCMAKE_CXX_COMPILER=clang++ \
-            -DAIE_ENABLE_BINDINGS_PYTHON=ON \
             -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
             -DLLVM_EXTERNAL_LIT=`pwd`/../llvm/build/bin/llvm-lit
           make -j$(nproc)

.github/workflows/generateDocs.yml

@@ -76,7 +76,6 @@ jobs:
             -DCMAKE_LINKER=lld \
             -DCMAKE_C_COMPILER=clang \
             -DCMAKE_CXX_COMPILER=clang++ \
-            -DAIE_ENABLE_BINDINGS_PYTHON=ON \
             -DAIE_INCLUDE_DOCS=ON \
             -DLLVM_EXTERNAL_LIT=`pwd`/../llvm/build/bin/llvm-lit
           make docs

CMakeLists.txt

@@ -51,7 +51,6 @@ option(LLVM_INCLUDE_TOOLS "Generate build targets for the LLVM tools." ON)
 option(LLVM_BUILD_TOOLS "Build the LLVM tools. If OFF, just generate build targets." ON)
 option(AIE_INCLUDE_INTEGRATION_TESTS
        "Generate build targets for the mlir-aie integration tests." OFF)
-option(AIE_ENABLE_BINDINGS_PYTHON "Enable MLIR python bindings." OFF)
 
 find_package(MLIR REQUIRED CONFIG)
 find_package(Boost REQUIRED)
@@ -123,39 +122,31 @@ if (AIE_INCLUDE_DOCS)
 endif()
 
 # python install directory
-if (AIE_ENABLE_BINDINGS_PYTHON)
-
-  if(NOT AIE_PYTHON_PACKAGES_DIR)
-    set(AIE_PYTHON_PACKAGES_DIR "${CMAKE_CURRENT_BINARY_DIR}/python")
-  endif()
-
-  # python install directory
-  if(NOT AIE_PYTHON_INSTALL_DIR)
-    set(AIE_PYTHON_INSTALL_DIR "python")
-  endif()
-
-  # setup python
-  find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
-  include(MLIRDetectPythonEnv)
-  mlir_detect_pybind11_install()
-  find_package(pybind11 2.6 REQUIRED)
+if(NOT AIE_PYTHON_PACKAGES_DIR)
+  set(AIE_PYTHON_PACKAGES_DIR "${CMAKE_CURRENT_BINARY_DIR}/python")
+endif()
 
+# python install directory
+if(NOT AIE_PYTHON_INSTALL_DIR)
+  set(AIE_PYTHON_INSTALL_DIR "python")
 endif()
 
+# setup python
+find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+include(MLIRDetectPythonEnv)
+mlir_detect_pybind11_install()
+find_package(pybind11 2.6 REQUIRED)
+
 add_subdirectory(include)
 add_subdirectory(lib)
-
+add_subdirectory(python)
 add_subdirectory(runtime_lib)
 add_subdirectory(aie_runtime_lib)
 add_subdirectory(reference_designs)
 add_subdirectory(test)
 add_subdirectory(tutorials)
 add_subdirectory(tools)
 
-if (AIE_ENABLE_BINDINGS_PYTHON)
-  add_subdirectory(python)
-endif()
-
 if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
   install(DIRECTORY include/aie
     DESTINATION include

python/CMakeLists.txt

@@ -89,4 +89,5 @@ add_mlir_python_modules(AiePythonModules
     AieAggregateCAPI
 )
 
-#add_subdirectory(test)
+add_dependencies(AiePythonModules AieCompilerPythonModules)
+add_subdirectory(aie/compiler)

python/aie/compiler/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Copyright (C) 2023, Advanced Micro Devices, Inc. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+declare_mlir_python_sources(AieCompilerPythonSources)
+
+declare_mlir_python_sources(AieCompilerPythonSources.Core
+  ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}"
+  ADD_TO_PARENT AieCompilerPythonSources
+  SOURCES_GLOB
+    aiecc/*.py
+    *.py
+)
+
+add_mlir_python_modules(AieCompilerPythonModules
+  ROOT_PREFIX "${AIE_PYTHON_PACKAGES_DIR}/aie/compiler"
+  INSTALL_PREFIX "${AIE_PYTHON_INSTALL_DIR}/aie/compiler"
+  DECLARED_SOURCES AieCompilerPythonSources
+  )

tools/aiecc/aiecc/cl_arguments.py → python/aie/compiler/aiecc/cl_arguments.py

@@ -11,9 +11,11 @@
 import sys
 import shutil
 
-from aiecc.configure import *
+from aie.compiler.aiecc.configure import *
 
-def parse_args():
+def parse_args(args=None):
+    if (args is None):
+        args = sys.argv[1:]
     parser = argparse.ArgumentParser(prog='aiecc')
     parser.add_argument('filename',
             metavar="file",
@@ -139,9 +141,7 @@ def parse_args():
             action='store_true',
             help='Show progress visualization')
 
-
-    opts = parser.parse_args(sys.argv[1:])
-
+    opts = parser.parse_args(args)
     return opts
 
 def strip_host_args_for_aiesim(args):

tools/aiecc/aiecc/main.py → python/aie/compiler/aiecc/main.py

@@ -21,8 +21,8 @@
 import timeit
 import asyncio
 
-import aiecc.cl_arguments
-import aiecc.configure
+import aie.compiler.aiecc.cl_arguments
+import aie.compiler.aiecc.configure
 
 import rich.progress as progress
 import re
@@ -140,8 +140,8 @@ async def chesshack(self, task, llvmir):
 
   async def prepare_for_chesshack(self, task):
       if(opts.compile and opts.xchesscc):
-        thispath = os.path.dirname(os.path.realpath(__file__))
-        runtime_lib_path = os.path.join(thispath, '..','..','aie_runtime_lib')
+        install_path = aie.compiler.aiecc.configure.install_path()
+        runtime_lib_path = os.path.join(install_path, 'aie_runtime_lib')
         chess_intrinsic_wrapper_cpp = os.path.join(runtime_lib_path, self.aie_target.upper(),'chess_intrinsic_wrapper.cpp')
 
         self.chess_intrinsic_wrapper = os.path.join(self.tmpdirname, 'chess_intrinsic_wrapper.ll')
@@ -157,8 +157,8 @@ async def process_core(self, core):
       if(self.stopall):
         return
 
-      thispath = os.path.dirname(os.path.realpath(__file__))
-      runtime_lib_path = os.path.join(thispath, '..','..','aie_runtime_lib', self.aie_target.upper())
+      install_path = aie.compiler.aiecc.configure.install_path()
+      runtime_lib_path = os.path.join(install_path, 'aie_runtime_lib', self.aie_target.upper())
       clang_path = os.path.dirname(shutil.which('clang'))
       # The build path for libc can be very different from where it's installed.
       llvmlibc_build_lib_path = os.path.join(clang_path, '..', 'runtimes', 'runtimes-' + self.aie_target.lower() + '-none-unknown-elf-bins', 'libc', 'lib', 'libc.a')
@@ -256,8 +256,8 @@ async def process_host_cgen(self):
       cmd = ['clang++','-std=c++11']
       if(opts.host_target):
         cmd += ['--target=%s' % opts.host_target]
-        if(opts.aiesim and opts.host_target != aiecc.configure.host_architecture):
-          sys.exit("Host cross-compile from " + aiecc.configure.host_architecture +
+        if(opts.aiesim and opts.host_target != aie.compiler.aiecc.configure.host_architecture):
+          sys.exit("Host cross-compile from " + aie.compiler.aiecc.configure.host_architecture +
                    " to --target=" + opts.host_target + " is not supported with --aiesim")
 
       if(self.opts.sysroot):
@@ -272,11 +272,11 @@ async def process_host_cgen(self):
         if(opts.host_target == 'aarch64-linux-gnu'):
           cmd += ['--gcc-toolchain=%s/usr' % opts.sysroot]
 
-      thispath = os.path.dirname(os.path.realpath(__file__))
-      runtime_xaiengine_path = os.path.join(thispath, '..','..','runtime_lib', opts.host_target.split('-')[0], 'xaiengine')
+      install_path = aie.compiler.aiecc.configure.install_path()
+      runtime_xaiengine_path = os.path.join(install_path, 'runtime_lib', opts.host_target.split('-')[0], 'xaiengine')
       xaiengine_include_path = os.path.join(runtime_xaiengine_path, "include")
       xaiengine_lib_path = os.path.join(runtime_xaiengine_path, "lib")
-      runtime_testlib_path = os.path.join(thispath, '..','..','runtime_lib', opts.host_target.split('-')[0], 'test_lib', 'lib')
+      runtime_testlib_path = os.path.join(install_path, 'runtime_lib', opts.host_target.split('-')[0], 'test_lib', 'lib')
       memory_allocator = os.path.join(runtime_testlib_path, 'libmemory_allocator_ion.a')
 
       cmd += [memory_allocator]
@@ -299,7 +299,7 @@ async def gen_sim(self, task):
       # For simulation, we need to additionally parse the 'remaining' options to avoid things
       # which conflict with the options below (e.g. -o)
       print(opts.host_args)
-      host_opts = aiecc.cl_arguments.strip_host_args_for_aiesim(opts.host_args)
+      host_opts = aie.compiler.aiecc.cl_arguments.strip_host_args_for_aiesim(opts.host_args)
 
       sim_dir = os.path.join(self.tmpdirname, 'sim')
       shutil.rmtree(sim_dir, ignore_errors=True)
@@ -314,11 +314,11 @@ def make_sim_dir(x):
       except FileExistsError:
         pass
 
-      thispath = os.path.dirname(os.path.realpath(__file__))
+      install_path = aie.compiler.aiecc.configure.install_path()
 
-      runtime_simlib_path = os.path.join(thispath, '..','..','aie_runtime_lib', self.aie_target.upper(),'aiesim')
-      runtime_testlib_path = os.path.join(thispath, '..','..','runtime_lib', opts.host_target.split('-')[0], 'test_lib', 'lib')
-      runtime_testlib_include_path = os.path.join(thispath, '..','..','runtime_lib', opts.host_target.split('-')[0], 'test_lib', 'include')
+      runtime_simlib_path = os.path.join(install_path, 'aie_runtime_lib', self.aie_target.upper(),'aiesim')
+      runtime_testlib_path = os.path.join(install_path, 'runtime_lib', opts.host_target.split('-')[0], 'test_lib', 'lib')
+      runtime_testlib_include_path = os.path.join(install_path, 'runtime_lib', opts.host_target.split('-')[0], 'test_lib', 'include')
       sim_makefile   = os.path.join(runtime_simlib_path, "Makefile")
       sim_genwrapper = os.path.join(runtime_simlib_path, "genwrapper_for_ps.cpp")
       file_physical = os.path.join(self.tmpdirname, 'input_physical.mlir')
@@ -342,7 +342,7 @@ def make_sim_dir(x):
                  memory_allocator
                  ]
 
-      # runtime_xaiengine_path = os.path.join(thispath, '..','..','runtime_lib', opts.host_target.split('-')[0], 'xaiengine')
+      # runtime_xaiengine_path = os.path.join(install_path,'runtime_lib', opts.host_target.split('-')[0], 'xaiengine')
       # xaiengine_lib_path = os.path.join(runtime_xaiengine_path, "lib")
       # '-L%s' % xaiengine_lib_path,
 
@@ -481,14 +481,11 @@ def dumpprofile(self):
 
 def main(builtin_params={}):
     global opts
-    opts = aiecc.cl_arguments.parse_args()
+    opts = aie.compiler.aiecc.cl_arguments.parse_args()
 
     is_windows = platform.system() == 'Windows'
 
-    thispath = os.path.dirname(os.path.realpath(__file__))
-
-    # Assume that aie-opt, etc. binaries are relative to this script.
-    aie_path = os.path.join(thispath, '..')
+    aie_path = aie.compiler.aiecc.configure.install_path()
     peano_path = os.path.join(opts.peano_install_dir, 'bin')
 
     if('VITIS' not in os.environ):

test/CMakeLists.txt

@@ -119,11 +119,7 @@ else()
 set(CONFIG_HAS_LIBXAIE 0)
 endif()
 
-if (AIE_ENABLE_BINDINGS_PYTHON)
-  set(ENABLE_PYTHON_TESTS 1)
-else()
-  set(ENABLE_PYTHON_TESTS 0)
-endif()
+set(ENABLE_PYTHON_TESTS 1)
 
 configure_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in

test/lit.cfg.py

@@ -26,6 +26,8 @@
 config.name = 'AIE'
 
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.environment['PYTHONPATH'] \
+    = "{}".format(os.path.join(config.aie_obj_root, "python"))
 
 # suffixes: A list of file extensions to treat as test files.
 config.suffixes = ['.mlir']

tools/aiecc/CMakeLists.txt

@@ -5,18 +5,10 @@
 #
 # (c) Copyright 2021 Xilinx Inc.
 
-set(PYTHON_INSTALL_PATH ${CMAKE_INSTALL_PREFIX}/bin)
-
-set(AIECC_SUBFILES
-  cl_arguments.py
-  __init__.py
-  main.py)
+set(AIECC_INSTALL_PATH ${CMAKE_INSTALL_PREFIX}/bin)
 
 set(AIECC_FILES
-  aiecc.py
-  aiecc/cl_arguments.py
-  aiecc/__init__.py
-  aiecc/main.py)
+  aiecc.py)
 
 set(AIECC_TARGETS ${AIECC_FILES})
 list(TRANSFORM AIECC_TARGETS PREPEND ${PROJECT_BINARY_DIR}/bin/)
@@ -53,8 +45,8 @@ endif()
 
 set(CONFIG_PEANO_INSTALL_DIR ${PEANO_INSTALL_DIR})
 
-configure_file(aiecc/configure.py.in ${PROJECT_BINARY_DIR}/bin/aiecc/configure.py)
-install(PROGRAMS ${PROJECT_BINARY_DIR}/bin/aiecc/configure.py DESTINATION ${PYTHON_INSTALL_PATH}/aiecc)
+configure_file(aiecc/configure.py.in ${PROJECT_BINARY_DIR}/python/aie/compiler/aiecc/configure.py)
+install(PROGRAMS ${PROJECT_BINARY_DIR}/python/aie/compiler/aiecc/configure.py DESTINATION ${AIE_PYTHON_INSTALL_DIR}/aie/compiler/aiecc)
 
 # Stuff our python into the build area:
 add_custom_target(aiecc.py ALL DEPENDS ${AIECC_TARGETS})
@@ -69,15 +61,6 @@ GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)")
 add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/bin/aiecc.py COMMAND
 ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/copy_aiecc.cmake)
 
-foreach(file ${AIECC_SUBFILES})
-add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/bin/aiecc/${file}
-                   COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/aiecc/${file}
-                   ${PROJECT_BINARY_DIR}/bin/aiecc/${file}
-                   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/aiecc/${file})
-endforeach()
 
 # Install our python stuff too.
-install(PROGRAMS aiecc.py DESTINATION ${PYTHON_INSTALL_PATH})
-foreach(file ${AIECC_SUBFILES})
-  install(PROGRAMS aiecc/${file} DESTINATION ${PYTHON_INSTALL_PATH}/aiecc)
-endforeach()
+install(PROGRAMS aiecc.py DESTINATION ${AIECC_INSTALL_PATH})

tools/aiecc/aiecc.py

@@ -6,7 +6,7 @@
 #
 # (c) Copyright 2021 Xilinx Inc.
 
-from aiecc.main import main
+from aie.compiler.aiecc.main import main
 
 if __name__ == '__main__':
     main()

tools/aiecc/aiecc/configure.py.in

@@ -5,11 +5,18 @@
 #
 # (c) Copyright 2021 Xilinx Inc.
 
+import os
+
 aie_link_with_xchesscc = @CONFIG_LINK_WITH_XCHESSCC@
 aie_compile_with_xchesscc = @CONFIG_COMPILE_WITH_XCHESSCC@
 aie_disable_link = @CONFIG_DISABLE_LINK@
 aie_disable_compile = @CONFIG_DISABLE_COMPILE@
 aie_unified_compile = True
 host_disable_compile = @CONFIG_DISABLE_HOST_COMPILE@
 peano_install_dir = "@CONFIG_PEANO_INSTALL_DIR@"
-host_architecture = "@LLVM_HOST_TRIPLE@"
+host_architecture = "@LLVM_HOST_TRIPLE@"
+
+def install_path():
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.join(path, '..', '..', '..', '..')
+    return os.path.realpath(path)

utils/build-mlir-aie-cross.sh

@@ -53,7 +53,6 @@ cmake -GNinja \
     -DMLIR_DIR=${LLVM_BUILD_DIR}/lib/cmake/mlir \
     -DLLVM_USE_LINKER=lld \
     -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-    -DAIE_ENABLE_BINDINGS_PYTHON=ON \
     -DCMAKE_BUILD_TYPE=Debug \
     -Wno-dev \
     .. |& tee cmake.log

utils/build-mlir-aie.sh

@@ -45,7 +45,6 @@ cmake -GNinja\
     -DCMAKE_INSTALL_PREFIX="../${INSTALL_DIR}" \
     -DCMAKE_BUILD_TYPE=Release \
     -DLLVM_ENABLE_ASSERTIONS=ON \
-    -DAIE_ENABLE_BINDINGS_PYTHON=ON \
     "-DAIE_RUNTIME_TARGETS=x86_64;aarch64" \
     -DAIE_RUNTIME_TEST_TARGET=aarch64 \
     .. |& tee cmake.log