-
Notifications
You must be signed in to change notification settings - Fork 68
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Parallelize the matrix build process
- Loading branch information
1 parent
7720f6e
commit 2edb3b2
Showing
4 changed files
with
289 additions
and
50 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,193 @@ | ||
| """ | ||
| Module where all functionality that purely relates to how we parallelize matrix_build.py | ||
| should live. It's not a perfect split of course, but it helps to separate the 'matrix' | ||
| logic from the 'how we build' logic. | ||
| """ | ||
| import os | ||
| import shutil | ||
| import subprocess | ||
| import tempfile | ||
| import time | ||
| from pathlib import Path | ||
| from typing import Optional, List | ||
| from dataclasses import dataclass | ||
|
|
||
|
|
||
| @dataclass | ||
| class Executor: | ||
| """ | ||
| Core data that defines a solution that is being built | ||
| """ | ||
| # The directory where we are building the solution | ||
| proj_dir: Path | ||
| # The solution dictionary | ||
| solution: Optional[dict] = None | ||
| # The process building the solution | ||
| proc: Optional[subprocess.Popen] = None | ||
| # Object that holds tempdir data, so that it can be cleaned up later | ||
| tempdir_obj: Optional[tempfile.TemporaryDirectory] = None | ||
|
|
||
|
|
||
| def generate_config_file(project_location: Path, flag_values: dict): | ||
| content = "#pragma once\n\n" | ||
| for key, value in flag_values.items(): | ||
| content += "#define {} {}\n".format(key, value) | ||
|
|
||
| with open(Path(project_location, "Configuration_local_matrix.hpp"), 'w') as f: | ||
| f.write(content) | ||
| f.flush() | ||
|
|
||
|
|
||
| def execute(project_location: Path, board: str, flag_values: dict, jobs: int = 1, out_pipe=True) -> subprocess.Popen: | ||
| """ | ||
| Start up an executor that is building a solution | ||
| :param project_location: The directory where to build the solution | ||
| :param board: The board type (aka environment) | ||
| :param flag_values: Dictionary of #defines to create a config file from | ||
| :param jobs: How many jobs the build process should use | ||
| :param out_pipe: If the executor's stdout/stderr should be pipes | ||
| :return: Process object that is executing the solution | ||
| """ | ||
| build_env = dict(os.environ) | ||
| build_env["PLATFORMIO_BUILD_FLAGS"] = "-DMATRIX_LOCAL_CONFIG=1" | ||
| generate_config_file(project_location, flag_values) | ||
|
|
||
| proc = subprocess.Popen( | ||
| ['pio', | ||
| 'run', | ||
| f'--project-dir={str(project_location.resolve())}', | ||
| f'--environment={board}', | ||
| f'--jobs={jobs}', | ||
| ], | ||
| stdout=subprocess.PIPE if out_pipe else None, | ||
| stderr=subprocess.PIPE if out_pipe else None, | ||
| env=build_env, | ||
| close_fds=True, | ||
| ) | ||
| return proc | ||
|
|
||
|
|
||
| def get_available_executor_idx(e_list: List[Executor]) -> Optional[int]: | ||
| """ | ||
| Get the index of an idle executor | ||
| :param e_list: List of executors | ||
| :return: Idle executor index, else None if all are busy | ||
| """ | ||
| for i, executor in enumerate(e_list): | ||
| if executor.proc is None: | ||
| return i | ||
| return None | ||
|
|
||
|
|
||
| def get_finished_executor_idx(e_list: List[Executor]) -> Optional[int]: | ||
| """ | ||
| Get the index of a finished executor | ||
| :param e_list: List of executors | ||
| :return: Finished executor index, else None if all are busy | ||
| """ | ||
| for i, executor in enumerate(e_list): | ||
| if executor.proc is not None and executor.proc.poll() is not None: | ||
| return i | ||
| return None | ||
|
|
||
|
|
||
| def cleanup_tempdirs(e_list: List[Executor]): | ||
| """ | ||
| Delete all the temporary directories that executors were using | ||
| :param e_list: List of executors | ||
| """ | ||
| for executor in e_list: | ||
| if executor.tempdir_obj is not None: | ||
| tempdir_path = executor.tempdir_obj.name | ||
| print(f'Deleting {tempdir_path}') | ||
| shutil.rmtree(tempdir_path, ignore_errors=True) | ||
|
|
||
|
|
||
| def create_executors(num_executors: int, local_paths_to_link: List[Path]) -> List[Executor]: | ||
| """ | ||
| Create a number of executors and their associated temporary directories, then | ||
| soft-link all needed project files | ||
| :param num_executors: Number of executors to create | ||
| :param local_paths_to_link: List of files to soft-link into the executor projects | ||
| :return: List of executors | ||
| """ | ||
| executor_list: List[Executor] = [] | ||
| print(f'Creating {num_executors} executors') | ||
| for executor_idx in range(num_executors): | ||
| tempdir = tempfile.TemporaryDirectory() | ||
| temp_proj_path = Path(tempdir.name) | ||
| for local_path in local_paths_to_link: | ||
| temp_dst_path = Path(temp_proj_path, local_path).resolve() | ||
| os.makedirs(temp_dst_path.parent, exist_ok=True) | ||
| os.symlink(local_path.resolve(), temp_dst_path) | ||
| executor_list.append(Executor(temp_proj_path, tempdir_obj=tempdir)) | ||
| print(f'{executor_idx} ', end='') | ||
| print() | ||
| return executor_list | ||
|
|
||
|
|
||
| def copy_caches_to_executors(src_proj_dir: Path, dst_executors: List[Executor]): | ||
| """ | ||
| Copy cache directories from a source directory to a number of executor project directories | ||
| :param src_proj_dir: Directory to copy from | ||
| :param dst_executors: List of executors to copy to | ||
| """ | ||
| print('Copying caches to other executors') | ||
| dir_names_to_copy = ['.pio', 'build_cache'] | ||
| for dir_name_to_copy in dir_names_to_copy: | ||
| src_path = Path(src_proj_dir, dir_name_to_copy) | ||
| for dst_executor in dst_executors: | ||
| dst_path = Path(dst_executor.proj_dir, dir_name_to_copy) | ||
| shutil.copytree(src_path, dst_path) | ||
|
|
||
|
|
||
| def get_source_files_to_link() -> List[Path]: | ||
| """ | ||
| Create a list of the important files from the local project. I didn't want to | ||
| use git here, since that might not pick up untracked (but needed) files. | ||
| :return: List of source files that a project needs in order to compile | ||
| """ | ||
| local_proj_path = Path('.') | ||
| venv_dirs = list(local_proj_path.glob('*venv*/')) | ||
| # Don't link the .pio directory because the builds need to be independent | ||
| pio_dirs = list(local_proj_path.glob('*.pio*/')) | ||
| cmake_dirs = list(local_proj_path.glob('*cmake-build*/')) | ||
|
|
||
| local_dirs_to_not_link = [Path('.git/'), Path('build_cache/')] + venv_dirs + pio_dirs + cmake_dirs | ||
| local_filenames_to_not_link = [ | ||
| Path('Configuration_local.hpp'), | ||
| Path('Configuration_local_matrix.hpp'), | ||
| ] | ||
|
|
||
| local_paths_to_link = [] | ||
| for local_dir_str, local_subdirs, local_files in os.walk(local_proj_path): | ||
| local_dir_path = Path(local_dir_str) | ||
| dir_shouldnt_be_linked = any(d == local_dir_path or d in local_dir_path.parents for d in local_dirs_to_not_link) | ||
| if dir_shouldnt_be_linked: | ||
| continue | ||
| for local_file in local_files: | ||
| local_file_full_path = Path(local_dir_path, local_file) | ||
| file_shouldnt_be_linked = any(local_file_full_path == f for f in local_filenames_to_not_link) | ||
| if file_shouldnt_be_linked: | ||
| continue | ||
| local_paths_to_link.append(local_file_full_path) | ||
| return local_paths_to_link | ||
|
|
||
|
|
||
| def wait_for_executor_to_finish(executor_list: List[Executor], timeout=0.1, poll_time=0.2): | ||
| """ | ||
| Block until an executor has finished building | ||
| :param executor_list: List of executors | ||
| :param timeout: Time to communicate() with the running process (kind of a hack) | ||
| :param poll_time: Time to wait before checking all executors again | ||
| """ | ||
| while get_finished_executor_idx(executor_list) is None: | ||
| for e in executor_list: | ||
| if e.proc is not None and e.proc.poll() is None: | ||
| # Communicate with the running processes to stop them from blocking | ||
| # (i.e. they spew too much output) | ||
| try: | ||
| _ = e.proc.communicate(timeout=timeout) | ||
| except subprocess.TimeoutExpired: | ||
| pass # This is expected and what should happen | ||
| time.sleep(poll_time) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.