Merge pull request #1258 from mrapp-ke/compile-libomp

Compile libomp from source on macOS

.changelog-bugfix.md

@@ -1,5 +1,6 @@
 # Quality-of-Life Improvements
 
+- When building macOS packages, we do now manually compile OpenMP to ensure that the bundled library matches the target platform.
 - The build system now uses a lightweight custom implementation instead of [SCons](https://scons.org/) and is better modularized to avoid unnecessary runs of Continuous Integration jobs when only certain parts of it are modified.
 - Releases are now automated via Continuous Integration, including the update of the project's changelog.
 - The presentation of algorithmic parameters in the documentation has been improved.

.github/workflows/template_build_macos.yml

@@ -18,13 +18,13 @@ jobs:
         with:
           python-version-file: .version-python
       - name: Install OpenMP
-        run: brew install libomp
+        run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} ./build dependency_libomp
       - name: Install OpenCL
         run: brew install opencl-clhpp-headers
       - name: Prepare ccache
         uses: hendrikmuhs/ccache-action@v1
         with:
           key: ${{ runner.os }}-test-build-ccache-${{ inputs.subproject }}
       - name: Compile via Clang
-        run: SUBPROJECTS=${{ inputs.subproject }} TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/opt/homebrew/opt/libomp/include/:/opt/homebrew/opt/opencl-clhpp-headers/include/
-          LIBRARY_PATH=/opt/homebrew/opt/libomp/lib/ ./build compile
+        run: SUBPROJECTS=${{ inputs.subproject }} TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/Users/runner/work/MLRL-Boomer/MLRL-Boomer/libomp/include/:/opt/homebrew/opt/opencl-clhpp-headers/include/
+          LIBRARY_PATH=/Users/runner/work/MLRL-Boomer/MLRL-Boomer/libomp/lib/ ./build compile

.github/workflows/template_publish_non_native.yml

@@ -41,7 +41,7 @@ jobs:
         with:
           platforms: all
       - name: Build package
-        uses: pypa/cibuildwheel@v2.17
+        uses: pypa/cibuildwheel@v2.22
         env:
           CIBW_BEFORE_BUILD_LINUX: ./build --clean && SUBPROJECTS=${{ inputs.subproject }} TEST_SUPPORT=disabled GPU_SUPPORT=disabled
             ./build install

.github/workflows/template_publish_platform.yml

@@ -40,22 +40,23 @@ jobs:
       - name: Set version
         if: ${{ matrix.os == 'windows-latest' && inputs.dev_release == true }}
         run: .\build.bat apply_development_version
-      - name: Install OpenMP
-        if: ${{ matrix.os == 'macos-latest' }}
-        run: brew install libomp
       - name: Prepare MSVC
         if: ${{ matrix.os == 'windows-latest' }}
         uses: ilammy/msvc-dev-cmd@v1
       - name: Build package
-        uses: pypa/cibuildwheel@v2.17
+        uses: pypa/cibuildwheel@v2.22
         env:
+          CIBW_BEFORE_ALL_MACOS: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} ./build dependency_libomp
           CIBW_BEFORE_BUILD_LINUX: ./build --clean && SUBPROJECTS=${{ inputs.subproject }} TEST_SUPPORT=disabled GPU_SUPPORT=disabled
             ./build install
           CIBW_BEFORE_BUILD_MACOS: rm -rf venv && ./build --clean && SUBPROJECTS=${{ inputs.subproject }} TEST_SUPPORT=disabled
-            GPU_SUPPORT=disabled CPLUS_INCLUDE_PATH=/opt/homebrew/opt/libomp/include/ LIBRARY_PATH=/opt/homebrew/opt/libomp/lib/
+            GPU_SUPPORT=disabled CPLUS_INCLUDE_PATH=/Users/runner/work/MLRL-Boomer/MLRL-Boomer/libomp/include/ LIBRARY_PATH=/Users/runner/work/MLRL-Boomer/MLRL-Boomer/libomp/lib/
             ./build install
           CIBW_BEFORE_BUILD_WINDOWS: .\build.bat --clean && set SUBPROJECTS=${{ inputs.subproject }} && set TEST_SUPPORT=disabled
             && set GPU_SUPPORT=disabled && .\build.bat install
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: >
+            DYLD_LIBRARY_PATH=/Users/runner/work/MLRL-Boomer/MLRL-Boomer/libomp/lib/ delocate-wheel  --require-archs {delocate_archs}
+            -w {dest_dir} -v {wheel}
           CIBW_BUILD_FRONTEND: build
           CIBW_ARCHS: auto64
           CIBW_SKIP: pp* *musllinux*

.github/workflows/test_build.yml

@@ -40,6 +40,8 @@ jobs:
               - 'build_system/targets/compilation/*'
               - 'build_system/targets/packaging/*'
               - 'build_system/targets/testing/*'
+              - 'build_system/targets/dependencies/*'
+              - 'build_system/targets/dependencies/macos/*'
             cpp_common: &cpp_common
               - *build_files
               - 'build_system/targets/compilation/cpp/*'

.github/workflows/test_publish.yml

@@ -34,6 +34,8 @@ jobs:
               - 'build_system/util/**'
               - 'build_system/targets/*.py'
               - 'build_system/targets/packaging/*'
+              - 'build_system/targets/dependencies/*'
+              - 'build_system/targets/dependencies/macos/*'
       - name: Read Python version
         uses: juliangruber/read-file-action@v1
         id: python_version

build_system/core/targets.py

@@ -311,7 +311,7 @@ class PhonyTarget(Target):
     A phony target, which executes a certain action unconditionally.
     """
 
-    Function = Callable[[], None]
+    Function = Callable[[BuildUnit], None]
 
     class Runnable(ABC):
         """
@@ -391,7 +391,7 @@ def build(self, build_unit: BuildUnit) -> Target:
 
             def action(module_registry: ModuleRegistry):
                 for function in self.functions:
-                    function()
+                    function(build_unit)
 
                 for runnable in self.runnables:
                     modules = module_registry.lookup(runnable.module_filter)

build_system/targets/changelog/changelog.py

@@ -9,6 +9,7 @@
 from functools import cached_property
 from typing import List, Optional
 
+from core.build_unit import BuildUnit
 from util.io import TextFile
 from util.log import Log
 
@@ -351,56 +352,56 @@ def __update_changelog(release_type: ReleaseType, *changeset_files):
         ChangesetFile(changeset_file).clear()
 
 
-def validate_changelog_bugfix():
+def validate_changelog_bugfix(_: BuildUnit):
     """
     Validates the changelog file that lists bugfixes.
     """
     __validate_changeset(CHANGESET_FILE_BUGFIX)
 
 
-def validate_changelog_feature():
+def validate_changelog_feature(_: BuildUnit):
     """
     Validates the changelog file that lists new features.
     """
     __validate_changeset(CHANGESET_FILE_FEATURE)
 
 
-def validate_changelog_main():
+def validate_changelog_main(_: BuildUnit):
     """
     Validates the changelog file that lists major updates.
     """
     __validate_changeset(CHANGESET_FILE_MAIN)
 
 
-def update_changelog_main():
+def update_changelog_main(_: BuildUnit):
     """
     Updates the projects changelog when releasing bugfixes.
     """
     __update_changelog(ReleaseType.MAJOR, CHANGESET_FILE_MAIN, CHANGESET_FILE_FEATURE, CHANGESET_FILE_BUGFIX)
 
 
-def update_changelog_feature():
+def update_changelog_feature(_: BuildUnit):
     """
     Updates the project's changelog when releasing new features.
     """
     __update_changelog(ReleaseType.MINOR, CHANGESET_FILE_FEATURE, CHANGESET_FILE_BUGFIX)
 
 
-def update_changelog_bugfix():
+def update_changelog_bugfix(_: BuildUnit):
     """
     Updates the project's changelog when releasing major updates.
     """
     __update_changelog(ReleaseType.PATCH, CHANGESET_FILE_BUGFIX)
 
 
-def print_current_version():
+def print_current_version(_: BuildUnit):
     """
     Prints the project's current version.
     """
     return Log.info('%s', str(VersionFile().version))
 
 
-def print_latest_changelog():
+def print_latest_changelog(_: BuildUnit):
     """
     Prints the changelog of the latest release.
     """

build_system/targets/dependencies/github/actions.py

@@ -5,16 +5,14 @@
 """
 from dataclasses import dataclass, replace
 from functools import cached_property, reduce
-from os import environ
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Set
 
 from core.build_unit import BuildUnit
-from util.env import get_env
 from util.log import Log
 
 from targets.dependencies.github.modules import GithubWorkflowModule
-from targets.dependencies.github.pygithub import GithubApi
 from targets.dependencies.github.pyyaml import YamlFile
+from targets.dependencies.pygithub import GithubApi
 
 
 @dataclass
@@ -202,8 +200,6 @@ class WorkflowUpdater:
     Allows checking the versions of GitHub Actions used in multiple workflows and updating outdated ones.
     """
 
-    ENV_GITHUB_TOKEN = 'GITHUB_TOKEN'
-
     @dataclass
     class OutdatedAction:
         """
@@ -246,24 +242,14 @@ def __eq__(self, other: 'WorkflowUpdater.UpdatedAction') -> bool:
         def __hash__(self) -> int:
             return hash(self.updated)
 
-    @staticmethod
-    def __get_github_token() -> Optional[str]:
-        github_token = get_env(environ, WorkflowUpdater.ENV_GITHUB_TOKEN)
-
-        if not github_token:
-            Log.warning('No GitHub API token is set. You can specify it via the environment variable %s.',
-                        WorkflowUpdater.ENV_GITHUB_TOKEN)
-
-        return github_token
-
     def __query_latest_action_version(self, action: Action) -> ActionVersion:
         repository_name = action.repository
 
         try:
-            latest_tag = GithubApi(self.build_unit) \
-                .set_token(self.github_token) \
+            latest_tag = self.github_api \
                 .open_repository(repository_name) \
-                .get_latest_release_tag()
+                .get_latest_release() \
+                .tag_name
 
             if not latest_tag:
                 raise RuntimeError('No releases available')
@@ -291,7 +277,7 @@ def __init__(self, build_unit: BuildUnit, module: GithubWorkflowModule):
         self.build_unit = build_unit
         self.module = module
         self.version_cache = {}
-        self.github_token = WorkflowUpdater.__get_github_token()
+        self.github_api = GithubApi(build_unit).set_token_from_env()
 
     @cached_property
     def workflows(self) -> Set[Workflow]:

build_system/targets/dependencies/github/requirements.txt

@@ -1,2 +1 @@
-pygithub >= 2.5, < 2.6
 pyyaml >= 6.0, < 6.1

build_system/targets/dependencies/macos/__init__.py

@@ -0,0 +1,16 @@
+"""
+Author: Michael Rapp ([email protected])
+
+Defines targets and modules for compiling native library dependencies on macOS. This is necessary when building
+pre-built packages via "cibuildwheel", because the de facto package manager "homebrew" will install libraries that are
+compiled for the specific macOS version the build machine is running, instead of using the proper version for the target
+platform (see https://cibuildwheel.pypa.io/en/stable/faq/#missing-dependencies).
+"""
+from core.build_unit import BuildUnit
+from core.targets import PhonyTarget, TargetBuilder
+
+from targets.dependencies.macos.targets import compile_libomp
+
+TARGETS = TargetBuilder(BuildUnit.for_file(__file__)) \
+    .add_phony_target('dependency_libomp').set_functions(compile_libomp) \
+    .build()

build_system/targets/dependencies/macos/cmake.py

@@ -0,0 +1,20 @@
+"""
+Author: Michael Rapp ([email protected])
+
+Provides classes that allow to run the external program "cmake".
+"""
+from util.run import Program
+
+
+class Cmake(Program):
+    """
+    Allows to run the external program "cmake".
+    """
+
+    def __init__(self, *arguments: str):
+        """
+        :param arguments: Optional arguments to be passed to the program "cmake"
+        """
+        super().__init__('cmake', *arguments)
+        self.install_program(False)
+        self.print_arguments(True)

build_system/targets/dependencies/macos/curl.py

@@ -0,0 +1,41 @@
+"""
+Author: Michael Rapp ([email protected])
+
+Provides classes that allow to run the external program "curl".
+"""
+from typing import Optional
+
+from util.cmd import Command
+from util.run import Program
+
+
+class CurlDownload(Program):
+    """
+    Allows to run the external program "curl" for downloading a file from a specific URL.
+    """
+
+    def __init__(self,
+                 url: str,
+                 authorization_header: Optional[str] = None,
+                 file_name: Optional[str] = None,
+                 follow_redirects: bool = True):
+        """
+        :param url:                     The URL of the file to be downloaded
+        :param authorization_header:    The authorization header to be set or None, if no such header should be set
+        :param file_name:               The name of the file to be saved or None, if the original name should be used
+        :param follow_redirects:        True, if redirects should be followed, False otherwise
+        """
+        super().__init__('curl')
+        self.add_conditional_arguments(follow_redirects, '--location')
+        self.add_conditional_arguments(file_name is not None, '-o', file_name)
+        self.add_conditional_arguments(file_name is None, '-O')
+        self.add_arguments(url)
+        self.use_authorization = authorization_header is not None
+        self.add_conditional_arguments(self.use_authorization, '-H', 'Authorization: ' + str(authorization_header))
+        self.install_program(False)
+        self.print_arguments(not self.use_authorization)
+
+    def __str__(self) -> str:
+        print_options = Command.PrintOptions()
+        print_options.print_arguments = not self.use_authorization
+        return print_options.format(self)

build_system/targets/dependencies/macos/tar.py

@@ -0,0 +1,21 @@
+"""
+Author: Michael Rapp ([email protected])
+
+Provides classes that allow to run the external program "tar".
+"""
+from util.run import Program
+
+
+class TarExtract(Program):
+    """
+    Allows to run the external program "tar" for extracting a file.
+    """
+
+    def __init__(self, file_to_extract: str, into_directory: str):
+        """
+        :param file_to_extract: The path to the file to be extracted
+        :param into_directory:  The path to the directory where the extracted files should be stored
+        """
+        super().__init__('tar', '--extract', '--file', file_to_extract, '--directory', into_directory)
+        self.install_program(False)
+        self.print_arguments(True)