Merge pull request #1182 from mrapp-ke/merge-feature

Merge feature into main branch

.github/workflows/test_build.yml

@@ -11,10 +11,13 @@ jobs:
     name: Detect changes
     runs-on: ubuntu-latest
     outputs:
+      cpp_boosting: ${{ steps.filter.outputs.cpp_boosting }}
+      cpp_seco: ${{ steps.filter.outputs.cpp_seco }}
       cpp: ${{ steps.filter.outputs.cpp }}
       cpp_tests: ${{ steps.filter.outputs.cpp_tests }}
-      python: ${{ steps.filter.outputs.python }}
       python_tests: ${{ steps.filter.outputs.python_tests }}
+      tests_boosting: ${{ steps.filter.outputs.tests_boosting }}
+      tests_seco: ${{ steps.filter.outputs.tests_seco }}
       any: ${{ steps.filter.outputs.any }}
     steps:
       - name: Look up Git repository in cache
@@ -38,34 +41,83 @@ jobs:
               - 'build_system/main.py'
               - 'build_system/core/**'
               - 'build_system/util/**'
-              - 'build_system/targets/paths.py'
+              - 'build_system/targets/*.py'
               - 'build_system/targets/compilation/*'
               - 'build_system/targets/testing/*'
-            cpp: &cpp
+            cpp_common: &cpp_common
               - *build_files
               - 'build_system/targets/compilation/cpp/*'
+              - 'cpp/**/meson.*'
+              - 'cpp/subprojects/common/include/**'
+              - 'cpp/subprojects/common/src/**'
+            cpp_boosting: &cpp_boosting
+              - *cpp_common
+              - 'cpp/subprojects/boosting/include/**'
+              - 'cpp/subprojects/boosting/src/**'
+            cpp_seco: &cpp_seco
+              - *cpp_common
+              - 'cpp/subprojects/seco/include/**'
+              - 'cpp/subprojects/seco/src/**'
+            cpp:
+              - *cpp_boosting
+              - *cpp_seco
+            cpp_tests_common: &cpp_tests_common
+              - *cpp_common
               - 'build_system/targets/testing/cpp/*'
-              - 'cpp/**/include/**'
-              - 'cpp/**/src/**'
-              - '**/*.pxd'
-              - '**/*.pyx'
-              - '**/meson.build'
-            cpp_tests: &cpp_tests
-              - *cpp
-              - 'cpp/**/test/**'
-            python: &python
-              - *build_files
+              - 'cpp/subprojects/common/test/**'
+            cpp_tests_boosting: &cpp_tests_boosting
+              - *cpp_tests_common
+              - *cpp_boosting
+              - 'cpp/subprojects/boosting/test/**'
+            cpp_tests_seco: &cpp_tests_seco
+              - *cpp_tests_common
+              - *cpp_seco
+              - 'cpp/subprojects/seco/test/**'
+            cpp_tests:
+              - *cpp_tests_boosting
+              - *cpp_tests_seco
+            python_tests_common: &python_tests_common
+              - *cpp_tests_common
               - 'build_system/targets/compilation/cython/*'
               - 'build_system/targets/testing/python/*'
-              - 'python/**/requirements.txt'
-              - 'python/**/mlrl/**'
-            python_tests: &python_tests
-              - *python
-              - *cpp_tests
-              - 'python/**/tests/**' 
+              - 'python/**/meson.*'
+              - 'python/subprojects/common/pyproject.toml'
+              - 'python/subprojects/common/requirements.txt'
+              - 'python/subprojects/common/setup.py'
+              - 'python/subprojects/common/mlrl/**'
+              - 'python/subprojects/testbed/pyproject.toml'
+              - 'python/subprojects/testbed/requirements.txt'
+              - 'python/subprojects/testbed/setup.py'
+              - 'python/subprojects/testbed/mlrl/**'
+              - 'python/tests/mlrl/common/**'
+            python_tests_boosting: &python_tests_boosting
+              - *python_tests_common
+              - *cpp_tests_boosting
+              - 'python/subprojects/boosting/pyproject.toml'
+              - 'python/subprojects/boosting/requirements.txt'
+              - 'python/subprojects/boosting/setup.py'
+              - 'python/subprojects/boosting/mlrl/**'
+              - 'python/tests/mlrl/boosting/**'
+            python_tests_seco: &python_tests_seco
+              - *python_tests_common
+              - *cpp_tests_seco
+              - 'python/subprojects/seco/pyproject.toml'
+              - 'python/subprojects/seco/requirements.txt'
+              - 'python/subprojects/seco/setup.py'
+              - 'python/subprojects/seco/mlrl/**'
+              - 'python/tests/mlrl/seco/**'
+            python_tests:
+              - *python_tests_boosting
+              - *python_tests_seco
+            tests_boosting: &tests_boosting
+              - *cpp_tests_boosting
+              - *python_tests_boosting
+            tests_seco: &tests_seco
+              - *cpp_tests_seco
+              - *python_tests_seco
             any:
-              - *cpp_tests
-              - *python_tests
+              - *tests_boosting
+              - *tests_seco
       - name: Save Git repository to cache
         uses: actions/cache/save@v4
         if: success() || failure()
@@ -101,12 +153,18 @@ jobs:
         uses: hendrikmuhs/ccache-action@v1
         with:
           key: ${{ runner.os }}-test-build-ccache
+      - name: Include subproject "boosting"
+        if: ${{ needs.changes.outputs.tests_boosting == 'true' }}
+        run: echo "INCLUDED_PROJECTS=$(echo "${INCLUDED_PROJECTS}boosting,")" >> $GITHUB_ENV
+      - name: Include subproject "seco"
+        if: ${{ needs.changes.outputs.tests_seco == 'true' }}
+        run: echo "INCLUDED_PROJECTS=$(echo "${INCLUDED_PROJECTS}seco,")" >> $GITHUB_ENV
       - name: Compile via GCC
-        if: ${{ needs.changes.outputs.cpp == 'true' }}
-        run: TEST_SUPPORT=enabled ./build compile_cpp
+        if: ${{ needs.changes.outputs.any == 'true' }}
+        run: SUBPROJECTS=$INCLUDED_PROJECTS TEST_SUPPORT=enabled ./build compile_cpp
       - name: Test C++ code
         if: ${{ needs.changes.outputs.cpp_tests == 'true' }}
-        run: ./build tests_cpp
+        run: SUBPROJECTS=$INCLUDED_PROJECTS ./build tests_cpp
       - name: Publish C++ test results
         if: ${{ needs.changes.outputs.cpp_tests == 'true' }} && (success() || failure())
         uses: EnricoMi/publish-unit-test-result-action@v2
@@ -116,14 +174,14 @@ jobs:
             cpp/build/meson-logs/testlog.junit.xml
       - name: Test Python code
         if: ${{ needs.changes.outputs.python_tests == 'true' }}
-        run: ./build tests_python
+        run: SUBPROJECTS=$INCLUDED_PROJECTS ./build tests_python
       - name: Publish Python test results
         if: ${{ needs.changes.outputs.python_tests == 'true' }} && (success() || failure())
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:
           check_name: Python Test Results
           files: |
-            python/build/test-results/*.xml
+            python/tests/build/test-results/*.xml
   macos_build:
     needs: changes
     if: ${{ needs.changes.outputs.cpp == 'true' }}
@@ -151,8 +209,14 @@ jobs:
         uses: hendrikmuhs/ccache-action@v1
         with:
           key: ${{ runner.os }}-test-build-ccache
+      - name: Include subproject "boosting"
+        if: ${{ needs.changes.outputs.cpp_boosting == 'true' }}
+        run: echo "INCLUDED_PROJECTS=$(echo "${INCLUDED_PROJECTS}boosting,")" >> $GITHUB_ENV
+      - name: Include subproject "seco"
+        if: ${{ needs.changes.outputs.cpp_seco == 'true' }}
+        run: echo "INCLUDED_PROJECTS=$(echo "${INCLUDED_PROJECTS}seco,")" >> $GITHUB_ENV
       - name: Compile via Clang
-        run: TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/opt/homebrew/opt/libomp/include/:/opt/homebrew/opt/opencl-clhpp-headers/include/
+        run: SUBPROJECTS=$INCLUDED_PROJECTS TEST_SUPPORT=disabled CPLUS_INCLUDE_PATH=/opt/homebrew/opt/libomp/include/:/opt/homebrew/opt/opencl-clhpp-headers/include/
           LIBRARY_PATH=/opt/homebrew/opt/libomp/lib/ ./build compile
   windows_build:
     needs: changes
@@ -186,8 +250,15 @@ jobs:
           token: ${{ github.token }}
       - name: Prepare sccache
         uses: mozilla-actions/[email protected]
+      - name: Include subproject "boosting"
+        if: ${{ needs.changes.outputs.cpp_boosting == 'true' }}
+        run: echo "INCLUDED_PROJECTS=$(echo "${env:INCLUDED_PROJECTS}boosting,")" >> $env:GITHUB_ENV
+      - name: Include subproject "seco"
+        if: ${{ needs.changes.outputs.cpp_seco == 'true' }}
+        run: echo "INCLUDED_PROJECTS=$(echo "${env:INCLUDED_PROJECTS}seco,")" >> $env:GITHUB_ENV
       - name: Compile via MSVC
         run: |-
+          $env:SUBPROJECTS = "$env:INCLUDED_PROJECTS"
           $env:TEST_SUPPORT = "disabled"
           $env:INCLUDE += ";$($pwd.Path)\vcpkg\packages\opencl_x64-windows\include"
           $env:LIB += ";$($pwd.Path)\vcpkg\packages\opencl_x64-windows\lib"

.github/workflows/test_changelog.yml

@@ -36,7 +36,7 @@ jobs:
               - 'build_system/main.py'
               - 'build_system/core/**'
               - 'build_system/util/**'
-              - 'build_system/targets/paths.py'
+              - 'build_system/targets/*.py'
               - 'build_system/targets/versioning/*'
             bugfix:
               - *build_files

.github/workflows/test_doc.yml

@@ -37,15 +37,15 @@ jobs:
               - 'build_system/main.py'
               - 'build_system/core/**'
               - 'build_system/util/**'
-              - 'build_system/targets/paths.py'
+              - 'build_system/targets/*.py'
             cpp: &cpp
               - *build_files
               - 'build_system/targets/documentation/cpp/*'
               - 'cpp/**/include/**'
             python: &python
               - *build_files
               - 'build_system/targets/documentation/python/*'
-              - 'python/**/mlrl/**'
+              - 'python/subprojects/**/mlrl/**'
             doc: &doc
               - *build_files
               - 'build_system/targets/documentation/*'

.github/workflows/test_format.yml

@@ -36,7 +36,7 @@ jobs:
               - 'build_system/main.py'
               - 'build_system/core/**'
               - 'build_system/util/**'
-              - 'build_system/targets/paths.py'
+              - 'build_system/targets/*.py'
             cpp:
               - *build_files
               - 'build_system/targets/code_style/*'

.github/workflows/test_publish.yml

@@ -39,7 +39,7 @@ jobs:
               - 'build_system/main.py'
               - 'build_system/core/**'
               - 'build_system/util/**'
-              - 'build_system/targets/paths.py'
+              - 'build_system/targets/*.py'
               - 'build_system/targets/packaging/*'
       - name: Read Python version
         uses: juliangruber/read-file-action@v1

build_system/targets/compilation/cpp/targets.py

@@ -14,11 +14,12 @@
 from targets.compilation.build_options import BuildOptions, EnvBuildOption
 from targets.compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup
 from targets.compilation.modules import CompilationModule
+from targets.modules import SubprojectModule
 
 MODULE_FILTER = CompilationModule.Filter(FileType.cpp())
 
 BUILD_OPTIONS = BuildOptions() \
-        .add(EnvBuildOption(name='subprojects')) \
+        .add(EnvBuildOption(name=SubprojectModule.ENV_SUBPROJECTS.lower())) \
         .add(EnvBuildOption(name='test_support', subpackage='common')) \
         .add(EnvBuildOption(name='multi_threading_support', subpackage='common')) \
         .add(EnvBuildOption(name='gpu_support', subpackage='common'))

build_system/targets/compilation/cython/targets.py

@@ -14,11 +14,12 @@
 from targets.compilation.build_options import BuildOptions, EnvBuildOption
 from targets.compilation.meson import MesonCompile, MesonConfigure, MesonInstall, MesonSetup
 from targets.compilation.modules import CompilationModule
+from targets.modules import SubprojectModule
 
 MODULE_FILTER = CompilationModule.Filter(FileType.cython())
 
 BUILD_OPTIONS = BuildOptions() \
-        .add(EnvBuildOption(name='subprojects'))
+        .add(EnvBuildOption(name=SubprojectModule.ENV_SUBPROJECTS.lower()))
 
 
 class SetupCython(BuildTarget.Runnable):

build_system/targets/dependencies/python/__init__.py

@@ -3,6 +3,8 @@
 
 Defines targets and modules for installing Python dependencies that are required by the project.
 """
+from os import path
+
 from core.build_unit import BuildUnit
 from core.targets import PhonyTarget, TargetBuilder
 
@@ -29,14 +31,9 @@
         root_directory=Project.BuildSystem.root_directory,
         requirements_file_search=Project.BuildSystem.file_search(),
     ),
-    PythonDependencyModule(
-        dependency_type=DependencyType.RUNTIME,
-        root_directory=Project.Python.root_directory,
-        requirements_file_search=Project.Python.file_search(),
-    ),
-    PythonDependencyModule(
-        dependency_type=DependencyType.BUILD_TIME,
-        root_directory=Project.Documentation.root_directory,
-        requirements_file_search=Project.Documentation.file_search(),
-    ),
+] + [
+    PythonDependencyModule(dependency_type=DependencyType.RUNTIME,
+                           root_directory=subproject,
+                           requirements_file_search=Project.Python.file_search())
+    for subproject in Project.Python.find_subprojects()
 ]

build_system/targets/dependencies/python/modules.py

@@ -4,11 +4,14 @@
 Implements modules that provide access to Python requirements files.
 """
 from enum import Enum
+from os import environ, path
 from typing import List
 
 from core.modules import Module
 from util.files import FileSearch
 
+from targets.modules import SubprojectModule
+
 
 class DependencyType(Enum):
     """
@@ -18,7 +21,7 @@ class DependencyType(Enum):
     RUNTIME = 'runtime'
 
 
-class PythonDependencyModule(Module):
+class PythonDependencyModule(SubprojectModule):
     """
     A module that provides access to Python requirements files.
     """
@@ -36,8 +39,11 @@ def __init__(self, *dependency_types: DependencyType):
             self.dependency_types = set(dependency_types)
 
         def matches(self, module: Module) -> bool:
-            return isinstance(module, PythonDependencyModule) and (not self.dependency_types
-                                                                   or module.dependency_type in self.dependency_types)
+            return isinstance(module, PythonDependencyModule) and (
+                not self.dependency_types
+                or module.dependency_type in self.dependency_types) and SubprojectModule.Filter.from_env(
+                    environ, always_match={SubprojectModule.SUBPROJECT_COMMON, SubprojectModule.SUBPROJECT_TESTBED
+                                           }).matches(module)
 
     def __init__(self,
                  dependency_type: DependencyType,
@@ -60,6 +66,10 @@ def find_requirements_files(self) -> List[str]:
         """
         return self.requirements_file_search.filter_by_name('requirements.txt').list(self.root_directory)
 
+    @property
+    def subproject_name(self) -> str:
+        return path.basename(self.root_directory)
+
     def __str__(self) -> str:
         return ('PythonDependencyModule {dependency_type="' + self.dependency_type.value + '", root_directory="'
                 + self.root_directory + '"}')

build_system/targets/documentation/cpp/__init__.py

@@ -28,11 +28,8 @@
 
 MODULES = [
     CppApidocModule(
-        root_directory=path.dirname(meson_file),
-        output_directory=path.join(Project.Documentation.apidoc_directory, 'cpp',
-                                   path.basename(path.dirname(meson_file))),
-        project_name=path.basename(path.dirname(meson_file)),
+        root_directory=subproject,
+        output_directory=path.join(Project.Documentation.apidoc_directory, 'cpp', path.basename(subproject)),
         include_directory_name='include',
-    ) for meson_file in Project.Cpp.file_search().filter_by_name('meson.build').list(Project.Cpp.root_directory)
-    if path.isdir(path.join(path.dirname(meson_file), 'include'))
+    ) for subproject in Project.Cpp.find_subprojects() if path.isdir(path.join(subproject, 'include'))
 ]

build_system/targets/documentation/cpp/breathe_apidoc.py

@@ -21,7 +21,7 @@ def __init__(self, build_unit: BuildUnit, module: CppApidocModule):
         :param build_unit:  The build unit from which the program should be run
         :param module:      The module, the program should be applied to
         """
-        super().__init__('breathe-apidoc', '--members', '--project', module.project_name, '-g', 'file', '-o',
+        super().__init__('breathe-apidoc', '--members', '--project', module.subproject_name, '-g', 'file', '-o',
                          module.output_directory, path.join(module.output_directory, 'xml'))
         self.module = module
         self.print_arguments(True)

build_system/targets/documentation/cpp/doxygen.py

@@ -32,10 +32,10 @@ class Doxygen(Program):
     @staticmethod
     def __create_environment(module: CppApidocModule) -> Dict:
         env = environ.copy()
-        set_env(env, 'DOXYGEN_PROJECT_NAME', 'libmlrl' + module.project_name)
+        set_env(env, 'DOXYGEN_PROJECT_NAME', 'libmlrl' + module.subproject_name)
         set_env(env, 'DOXYGEN_INPUT_DIR', module.include_directory)
         set_env(env, 'DOXYGEN_OUTPUT_DIR', module.output_directory)
-        set_env(env, 'DOXYGEN_PREDEFINED', 'MLRL' + module.project_name.upper() + '_API=')
+        set_env(env, 'DOXYGEN_PREDEFINED', 'MLRL' + module.subproject_name.upper() + '_API=')
         return env
 
     def __init__(self, build_unit: BuildUnit, module: CppApidocModule):