scons: SConscript/SConstruct refactor

SConstruct

@@ -55,53 +55,57 @@ AddOption('--minimal',
           default=os.path.islink(Dir('#laika/').abspath),
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-## Architecture name breakdown (arch)
-## - larch64: linux tici aarch64
-## - aarch64: linux pc aarch64
-## - x86_64:  linux pc x64
-## - Darwin:  mac x64 or arm64
-real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
-if platform.system() == "Darwin":
-  arch = "Darwin"
+# *** Target and Architecture
+
+## Target name breakdown (target)
+## - agnos-aarch64: linux tici aarch64
+## - linux-aarch64: linux pc aarch64
+## - linux-x86_64:  linux pc x64
+## - Darwin:        mac x64 or arm64
+arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+system = platform.system()
+if system == "Darwin":
+  target = "Darwin"
   brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-elif arch == "aarch64" and AGNOS:
-  arch = "larch64"
-assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
+elif system == "Linux":
+  if arch == "aarch64" and AGNOS:
+    target = "agnos-aarch64"
+  else:
+    target = f"linux-{arch}"
+else:
+  raise Exception(f"Unsupported platform: {system}")
+
+assert target in ["agnos-aarch64", "linux-aarch64", "linux-x86_64", "Darwin"]
 
+# *** Environment setup
 lenv = {
   "PATH": os.environ['PATH'],
-  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{target}/lib").abspath],
   "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
 
   "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
   "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{target}/t_renderer"
 }
-
 rpath = lenv["LD_LIBRARY_PATH"].copy()
 
-if arch == "larch64":
-  lenv["LD_LIBRARY_PATH"] += ['/data/data/com.termux/files/usr/lib']
-
+if target == "agnos-aarch64":
+  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
   cpppath = [
     "#third_party/opencl/include",
   ]
-
   libpath = [
+    "#third_party/acados/agnos-aarch64/lib",
+    "#third_party/snpe/agnos-aarch64",
+    "#third_party/libyuv/agnos-aarch64/lib",
     "/usr/local/lib",
     "/usr/lib",
     "/system/vendor/lib64",
-    f"#third_party/acados/{arch}/lib",
-  ]
-
-  libpath += [
-    "#third_party/snpe/larch64",
-    "#third_party/libyuv/larch64/lib",
     "/usr/lib/aarch64-linux-gnu"
   ]
-  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
-  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
   rpath += ["/usr/local/lib"]
+  lenv["LD_LIBRARY_PATH"] += ['/data/data/com.termux/files/usr/lib']
 else:
   cflags = []
   cxxflags = []
@@ -112,10 +116,10 @@ else:
   ]
 
   # MacOS
-  if arch == "Darwin":
+  if target == "Darwin":
     libpath = [
-      f"#third_party/libyuv/{arch}/lib",
-      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{target}/lib",
+      f"#third_party/acados/{target}/lib",
       f"{brew_prefix}/lib",
       f"{brew_prefix}/opt/[email protected]/lib",
       "/System/Library/Frameworks/OpenGL.framework/Libraries",
@@ -131,21 +135,19 @@ else:
   # Linux
   else:
     libpath = [
-      f"#third_party/acados/{arch}/lib",
-      f"#third_party/libyuv/{arch}/lib",
-      f"#third_party/mapbox-gl-native-qt/{arch}",
-      "#cereal",
-      "#common",
-      "/usr/lib",
+      f"#third_party/acados/{target}/lib",
+      f"#third_party/libyuv/{target}/lib",
+      f"#third_party/mapbox-gl-native-qt/{target}",
       "/usr/local/lib",
+      "/usr/lib",
     ]
 
-    if arch == "x86_64":
+    if target == "linux-x86_64":
       libpath += [
-        f"#third_party/snpe/{arch}"
+        f"#third_party/snpe/{target}"
       ]
       rpath += [
-        Dir(f"#third_party/snpe/{arch}").abspath,
+        Dir(f"#third_party/snpe/{target}").abspath,
       ]
 
 if GetOption('asan'):
@@ -158,11 +160,11 @@ else:
   ccflags = []
   ldflags = []
 
-# no --as-needed on mac linker
-if arch != "Darwin":
+## no --as-needed on mac linker
+if target != "Darwin":
   ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
 
-# Enable swaglog include in submodules
+## Enable swaglog include in submodules
 cflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 
@@ -221,19 +223,20 @@ env = Environment(
   tools=["default", "cython", "compilation_db"],
 )
 
-if arch == "Darwin":
-  # RPATH is not supported on macOS, instead use the linker flags
-  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
-  env["LINKFLAGS"] += darwin_rpath_link_flags
+## RPATH is not supported on macOS, instead use the linker flags
+if target == "Darwin":
+  Darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += Darwin_rpath_link_flags
 
 if GetOption('compile_db'):
   env.CompilationDatabase('compile_commands.json')
 
-# Setup cache dir
+## Setup cache dir
 cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
 CacheDir(cache_dir)
 Clean(["."], cache_dir)
 
+## Setup optional progress bar
 node_interval = 5
 node_count = 0
 def progress_function(node):
@@ -244,39 +247,25 @@ def progress_function(node):
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-SHARED = False
-
-# TODO: this can probably be removed
-def abspath(x):
-  if arch == 'aarch64':
-    pth = os.path.join("/data/pythonpath", x[0].path)
-    env.Depends(pth, x)
-    return File(pth)
-  else:
-    # rpath works elsewhere
-    return x[0].path.rsplit("/", 1)[1][:-3]
-
-# Cython build environment
+# *** Cython build environment
 py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
 envCython["CPPPATH"] += [py_include, np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
-if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
+if target == "Darwin":
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + Darwin_rpath_link_flags
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
-Export('envCython')
-
-# Qt build environment
+# *** Qt build environment
 qt_env = env.Clone()
 qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus", "Xml"]
 
 qt_libs = []
-if arch == "Darwin":
+if target == "Darwin":
   qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
   qt_dirs = [
     os.path.join(qt_env['QTDIR'], "include"),
@@ -296,15 +285,13 @@ else:
   ]
   qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
 
-  qt_libs = [f"Qt5{m}" for m in qt_modules]
-  if arch == "larch64":
+  qt_libs = [f"Qt5{m}" for m in qt_modules] + ["GL"]
+  if target == "agnos-aarch64":
     qt_libs += ["GLESv2", "wayland-client"]
-    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
-  elif arch != "Darwin":
-    qt_libs += ["GL"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/agnos-aarch64/bin/").abspath)
 qt_env['QT3DIR'] = qt_env['QTDIR']
 
-# compatibility for older SCons versions
+## compatibility for older SCons versions
 try:
   qt_env.Tool('qt3')
 except SCons.Errors.UserError:
@@ -337,34 +324,29 @@ if GetOption("clazy"):
   qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
   qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
 
-Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED')
+# *** Export environments and global variables
+Export('env', 'envCython', 'qt_env', 'target', 'system', 'arch')
 
+# *** Build common code
 SConscript(['common/SConscript'])
-Import('_common', '_gpucommon')
-
-if SHARED:
-  common, gpucommon = abspath(common), abspath(gpucommon)
-else:
-  common = [_common, 'json11']
-  gpucommon = [_gpucommon]
 
-Export('common', 'gpucommon')
-
-# cereal and messaging are shared with the system
+# *** Build cereal and messaging
 SConscript(['cereal/SConscript'])
-if SHARED:
-  cereal = abspath([File('cereal/libcereal_shared.so')])
-  messaging = abspath([File('cereal/libmessaging_shared.so')])
-else:
-  cereal = [File('#cereal/libcereal.a')]
-  messaging = [File('#cereal/libmessaging.a')]
-  visionipc = [File('#cereal/libvisionipc.a')]
-messaging_python = [File('#cereal/messaging/messaging_pyx.so')]
 
+cereal = [File('#cereal/libcereal.a')]
+messaging = [File('#cereal/libmessaging.a')]
+messaging_python = [File('#cereal/messaging/messaging_pyx.so')]
+visionipc = [File('#cereal/libvisionipc.a')]
 Export('cereal', 'messaging', 'messaging_python', 'visionipc')
 
-# Build rednose library and ekf models
+# *** Build other submodules
+SConscript([
+  'body/board/SConscript',
+  'opendbc/can/SConscript',
+  'panda/SConscript',
+])
 
+# *** Build rednose library and ekf models
 rednose_deps = [
   "#selfdrive/locationd/models/constants.py",
   "#selfdrive/locationd/models/gnss_helpers.py",
@@ -379,7 +361,7 @@ rednose_config = {
   },
 }
 
-if arch != "larch64":
+if target != "agnos-aarch64":
   rednose_config['to_build'].update({
     'loc_4': ('#selfdrive/locationd/models/loc_kf.py', True, [], rednose_deps),
     'lane': ('#selfdrive/locationd/models/lane_kf.py', True, [], rednose_deps),
@@ -391,30 +373,21 @@ if arch != "larch64":
 Export('rednose_config')
 SConscript(['rednose/SConscript'])
 
-# Build system services
+# *** Build system services
 SConscript([
   'system/clocksd/SConscript',
   'system/proclogd/SConscript',
   'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
-if arch != "Darwin":
+if target != "Darwin":
   SConscript([
     'system/camerad/SConscript',
     'system/sensord/SConscript',
     'system/logcatd/SConscript',
   ])
 
-# Build openpilot
-
-# build submodules
-SConscript([
-  'body/board/SConscript',
-  'cereal/SConscript',
-  'opendbc/can/SConscript',
-  'panda/SConscript',
-])
-
+# *** Build openpilot
 SConscript(['third_party/SConscript'])
 
 SConscript(['selfdrive/boardd/SConscript'])
@@ -425,7 +398,7 @@ SConscript(['selfdrive/navd/SConscript'])
 SConscript(['selfdrive/modeld/SConscript'])
 SConscript(['selfdrive/ui/SConscript'])
 
-if arch in ['x86_64', 'aarch64', 'Darwin'] and Dir('#tools/cabana/').exists() and GetOption('extras'):
+if (target in ['linux-x86_64', 'linux-aarch64', 'Darwin'] and Dir('#tools/cabana/').exists()) and GetOption('extras'):
   SConscript(['tools/replay/SConscript'])
   SConscript(['tools/cabana/SConscript'])
 

common/SConscript

@@ -1,10 +1,6 @@
-Import('env', 'envCython', 'arch', 'SHARED')
-
-if SHARED:
-  fxn = env.SharedLibrary
-else:
-  fxn = env.Library
+Import('env', 'envCython', 'target')
 
+# *** common
 common_libs = [
   'params.cc',
   'statlog.cc',
@@ -14,26 +10,24 @@ common_libs = [
   'watchdog.cc',
   'ratekeeper.cc'
 ]
-
-if arch != "Darwin":
+if target != "Darwin":
   common_libs.append('gpio.cc')
 
-_common = fxn('common', common_libs, LIBS="json11")
+common = env.Library('common', common_libs, LIBS="json11")
 
-files = [
+# *** gpucommon
+gpucommon_files = [
   'clutil.cc',
 ]
+gpucommon = env.Library('gpucommon', gpucommon_files)
 
-_gpucommon = fxn('gpucommon', files)
-Export('_common', '_gpucommon')
-
+# *** tests
 if GetOption('extras'):
-  env.Program('tests/test_common',
-              ['tests/test_runner.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc', 'tests/test_ratekeeper.cc'],
-              LIBS=[_common, 'json11', 'zmq', 'pthread'])
+  test_files = ['tests/test_runner.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc', 'tests/test_ratekeeper.cc']
+  env.Program('tests/test_common', test_files, LIBS=[common, 'json11', 'zmq', 'pthread'])
 
-# Cython bindings
-params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+# *** cython bindings
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [common, 'zmq', 'json11'])
 
 SConscript([
   'kalman/SConscript',
@@ -43,4 +37,4 @@ SConscript([
 Import('simple_kalman_python', 'transformations_python')
 common_python = [params_python, simple_kalman_python, transformations_python]
 
-Export('common_python')
+Export('common', 'gpucommon', 'common_python')