diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml index 8260499..3025817 100644 --- a/.github/workflows/build_all.yml +++ b/.github/workflows/build_all.yml @@ -100,8 +100,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -186,17 +186,17 @@ jobs: - name: Setup repos if: ${{ inputs.use_sha1_from_live }} - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} - name: Setup repos if: ${{ !inputs.use_sha1_from_live }} - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Launch build - run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt + run: | + python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt #- name: Push info to DB # run: | @@ -534,8 +534,8 @@ jobs: name: build_${{ matrix.device }}_10.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build" - name: Archive output file uses: actions/upload-artifact@v3 @@ -577,12 +577,12 @@ jobs: name: merged_build_stax.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build" - name: Convert to markdown run: | - python3 scripts/output_scripts/convert.py --input_file full_build_output.json --output_file out.md --key build + python3 scripts/entrypoint.py convert_output --input_file full_build_output.json --output_file out.md --key build cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -590,7 +590,7 @@ jobs: - name: Convert to slack json run: | - python3 scripts/output_scripts/slack.py --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} + python3 scripts/entrypoint.py slack_output --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -603,7 +603,7 @@ jobs: - name: Set job status run: | - python3 scripts/output_scripts/status.py --input_file full_build_output.json --key build + python3 scripts/entrypoint.py status_output --input_file full_build_output.json --key build - name: Archive output file if: always() diff --git a/.github/workflows/fast-checks.yml b/.github/workflows/fast-checks.yml new file mode 100644 index 0000000..99183cd --- /dev/null +++ b/.github/workflows/fast-checks.yml @@ -0,0 +1,48 @@ +name: Fast Python checks + +on: + workflow_dispatch: + push: + branches: + - develop + - master + pull_request: + +jobs: + lint: + strategy: + fail-fast: false + name: Linting + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + - run: pip install flake8 + - name: Flake8 lint Python code + run: find scripts/ -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+' + + mypy: + strategy: + fail-fast: false + name: Type checking + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + - run: pip install mypy types-requests + - name: Mypy type checking + run: mypy scripts/ + + misspell: + strategy: + fail-fast: false + name: Check misspellings + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + - name: Check misspellings + uses: codespell-project/actions-codespell@v1 + with: + builtin: clear,rare + check_filenames: true diff --git a/.github/workflows/refresh_inputs.yml b/.github/workflows/refresh_inputs.yml index bc18a5d..a1fc1bf 100644 --- a/.github/workflows/refresh_inputs.yml +++ b/.github/workflows/refresh_inputs.yml @@ -20,14 +20,14 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest - + steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Build db - run: | - python3 scripts/create_app_list/main.py --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} + run: | + python3 scripts/entrypoint.py create_app_list --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} - name: Upload file uses: actions/upload-artifact@v3 with: @@ -84,7 +84,7 @@ jobs: if: ${{ failure() && (github.event_name == 'schedule' || inputs.create_pr == true) }} run: | apk add github-cli - gh pr create --base "test-all-apps" --head "update-input" --title "[BOT] Update input file" --body "Input file has changed ! Please review changes !" + gh pr create --base "develop" --head "update-input" --title "[BOT] Update input file" --body "Input file has changed ! Please review changes !" env: GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/scan_all.yml b/.github/workflows/scan_all.yml index 5802697..a5ca7f2 100644 --- a/.github/workflows/scan_all.yml +++ b/.github/workflows/scan_all.yml @@ -83,8 +83,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -168,12 +168,12 @@ jobs: name: input_${{ matrix.index }}.json - name: Setup repos - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Launch scan - run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt + run: | + python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt #- name: Push info to DB # run: | @@ -254,8 +254,8 @@ jobs: name: scan_${{ matrix.device }}_10.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan" - name: Archive output file uses: actions/upload-artifact@v3 @@ -554,12 +554,12 @@ jobs: name: merged_scan_stax.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan" - name: Convert to markdown run: | - python3 scripts/output_scripts/convert.py --input_file full_scan_output.json --output_file out.md --key scan + python3 scripts/entrypoint.py convert_output --input_file full_scan_output.json --output_file out.md --key scan cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -567,7 +567,7 @@ jobs: - name: Convert to slack json run: | - python3 scripts/output_scripts/slack.py --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} + python3 scripts/entrypoint.py slack_output --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -580,7 +580,7 @@ jobs: - name: Set job status run: | - python3 scripts/output_scripts/status.py --input_file full_scan_output.json --key scan + python3 scripts/entrypoint.py status_output --input_file full_scan_output.json --key scan - name: Archive output file uses: actions/upload-artifact@v3 diff --git a/.github/workflows/test_all.yml b/.github/workflows/test_all.yml index 538333b..7e3b642 100644 --- a/.github/workflows/test_all.yml +++ b/.github/workflows/test_all.yml @@ -47,8 +47,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -136,8 +136,8 @@ jobs: name: input_${{ matrix.index }}.json - name: Setup repos - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Install tests dependencies run: | @@ -167,22 +167,22 @@ jobs: - name: Launch test nanosp if: ${{ env.test_nanosp }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt - name: Launch test nanox if: ${{ env.test_nanox }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt - name: Launch test stax if: ${{ env.test_stax }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt - name: Merge output files - run: | - python scripts/output_scripts/merge.py --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test" + run: | + python scripts/entrypoint.py merge_output --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test" - name: Archive output file uses: actions/upload-artifact@v3 with: @@ -339,8 +339,8 @@ jobs: name: test_output_10.json - name: Merge output files - run: | - python scripts/output_scripts/merge.py --input_pattern "test_output_*.json" --output_file test_output.json --key "test" + run: | + python scripts/entrypoint.py merge_output --input_pattern "test_output_*.json" --output_file test_output.json --key "test" - name: Archive output file uses: actions/upload-artifact@v3 @@ -351,7 +351,7 @@ jobs: - name: Convert to markdown run: | - python scripts/output_scripts/convert.py --input_file test_output.json --output_file out.md --key test + python scripts/entrypoint.py convert_output --input_file test_output.json --output_file out.md --key test cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -359,7 +359,7 @@ jobs: - name: Convert to slack json run: | - python scripts/output_scripts/slack.py --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }} + python scripts/entrypoint.py slack_output --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -372,4 +372,4 @@ jobs: - name: Set job status run: | - python scripts/output_scripts/status.py --input_file test_output.json --key test + python scripts/entrypoint.py status_output --input_file test_output.json --key test diff --git a/README.md b/README.md index 5b8d2c1..9185ebb 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ file generated previously. - `scan_all.yml`: Scans for selected devices - `refresh_inputs.yml`: Check whether the input list needs updating. -To reduce CI run times, the input file is automatically splitted into 10 sub-inputs, and then all the inputs are run through a matrix strategy. +To reduce CI run times, the input file is automatically split into 10 sub-inputs, and then all the inputs are run through a matrix strategy. ### 3. Planned Improvements - **Support for ZEMU Tests** @@ -58,5 +58,5 @@ Alternatively you can run the script from the actions tab of the repo. You can view the result in the summary of the GH action: :red_circle: means a fail. :heavy_check_mark: means success, -:fast_forward: means the action was explicitely blacklisted in the input file. +:fast_forward: means the action was explicitly blacklisted in the input file. nothing: means there was no variant associated to the device when running make listvariants. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..479d38e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.mypy] +ignore_missing_imports = true diff --git a/scripts/build_and_test/main.py b/scripts/build_and_test/__init__.py similarity index 58% rename from scripts/build_and_test/main.py rename to scripts/build_and_test/__init__.py index 3a1ca77..5bc7c84 100644 --- a/scripts/build_and_test/main.py +++ b/scripts/build_and_test/__init__.py @@ -1,51 +1,23 @@ import json from pathlib import Path -from argparse import ArgumentParser -from build_app import build_all_devices -from test_app import test_all_devices -from scan_app import scan_all_devices -from device import Devices +from argparse import Namespace + +from build_and_test.sha1 import override_sha1 +from build_and_test.build_app import build_all_devices +from build_and_test.test_app import test_all_devices +from build_and_test.scan_app import scan_all_devices +from build_and_test.device import Devices from utils import git_setup, merge_json -from sha1 import override_sha1 SDK_NAME = "sdk" SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" SDK_BRANCH = "origin/master" -if __name__ == "__main__": - parser = ArgumentParser() +def main(args: Namespace) -> None: input_json = {} - parser.add_argument("--skip_setup", action='store_true') - - # Devices - parser.add_argument("--all", action='store_true') - parser.add_argument("--nanos", action='store_true') - parser.add_argument("--nanosp", action='store_true') - parser.add_argument("--nanox", action='store_true') - parser.add_argument("--stax", action='store_true') - - parser.add_argument("--test", action='store_true') - parser.add_argument("--build", action='store_true') - parser.add_argument("--scan_build", action='store_true') - - parser.add_argument("--sdk_ref", required=False, type=Path, default="origin/master") - - parser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) - parser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) - parser.add_argument("--logs_file", required=False, type=Path, - default=Path("output_files/error_logs.txt")) - parser.add_argument("--workdir", required=False, type=str, default="workdir") - - parser.add_argument("--use_sha1_from_live", required=False, action='store_true') - parser.add_argument("--provider", required=False, type=str) - parser.add_argument("--device", required=False, type=str) - parser.add_argument("--version", required=False, type=str) - - args = parser.parse_args() - - abs_workdir = Path.cwd()/args.workdir + abs_workdir = Path.cwd() / args.workdir if not abs_workdir.exists(): abs_workdir.mkdir() @@ -90,19 +62,16 @@ input_json = override_sha1(input_json, args.provider, args.device, args.version) - git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) - output = {} - test_output = [] build_output = [] logs = "" for app_json in input_json: - repo_name = app_json.get("name") + repo_name = app_json["name"] if not args.skip_setup: - repo_ref = app_json.get("ref") - repo_url = app_json.get("url") + repo_ref = app_json["ref"] + repo_url = app_json["url"] print(f"Setup {repo_name}") git_setup(repo_name, repo_ref, repo_url, abs_workdir) @@ -124,7 +93,7 @@ build_output.append(scan_app) logs += log - output = merge_json(build_output, test_output, "name") + output = merge_json(build_output, [], "name") with open(args.output_file, 'w') as json_file: json.dump(output, json_file, indent=1) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 3a9f86a..7f23ed6 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -1,20 +1,29 @@ +import os from pathlib import Path -from device import Devices, Device +from typing import Dict, Optional, Tuple, Union + +from build_and_test.device import Devices, Device from utils import run_cmd -import os -def build_variant(target: str, sdk_path: str, variant_param: str, variant_value: str, app_build_path: - Path, extra_flags: str=""): + +def build_variant(target: str, + sdk_path: Path, + variant_param: Optional[str], + variant_value: str, + app_build_path: Path, + extra_flags: str = "") -> Tuple[int, str]: if not os.path.exists(app_build_path): print("\t=> KO") return True, f"Error: {app_build_path} does not exists\n" - error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) + run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) if variant_param: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", cwd=app_build_path, no_throw=True) + error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", + cwd=app_build_path, no_throw=True) else: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j {extra_flags}", cwd=app_build_path, no_throw=True) + error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j {extra_flags}", + cwd=app_build_path, no_throw=True) if error: print("\t=> KO") @@ -22,7 +31,11 @@ def build_variant(target: str, sdk_path: str, variant_param: str, variant_value: return error, log -def build_all_variants(target: str, sdk_path: str, variant_param: str, variant_list: list, app_build_path: Path): +def build_all_variants(target: str, + sdk_path: Path, + variant_param: Optional[str], + variant_list: list, + app_build_path: Path) -> Tuple[Dict[str, str], str]: output = {} error_log = "" for variant in variant_list: @@ -37,28 +50,36 @@ def build_all_variants(target: str, sdk_path: str, variant_param: str, variant_l return output, error_log -def build_device(device: Device, variant_param: str, app_build_path: Path, sdk_path: Path, app_json: dict): - blacklist = app_json.get("build_blacklist", "[]") +def build_device(device: Device, + variant_param: Optional[str], + app_build_path: Path, + sdk_path: Path, + app_json: dict) -> Tuple[Union[str, Dict[str, str]], str]: + blacklist = app_json.get("build_blacklist", []) error_log = "" if not device.selected: - return None, error_log + return "Unselected", error_log if device.model_name in blacklist: - return "Skipped", error_log + return "Blacklisted", error_log variants = app_json.get(f"variants_{device.model_name}", []) - variant_output = {} + variant_output: Dict[str, str] = {} if len(variants) > 0: - variant_output, error_log = build_all_variants(device.target_name, sdk_path, variant_param, variants, app_build_path) + variant_output, error_log = build_all_variants(device.target_name, + sdk_path, + variant_param, + variants, + app_build_path) return variant_output, error_log def build_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: Path): - repo_name = app_json.get("name") + repo_name = app_json["name"] variant_param = app_json.get("variant_param") - app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) + app_build_path = workdir / Path(repo_name + "/" + app_json.get("build_path", ".")) output = { "name": repo_name, @@ -73,13 +94,13 @@ def build_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output, stax_log = build_device(devices.stax, variant_param, app_build_path, sdk_path, app_json) - if nanos_output: + if nanos_output and devices.nanos.selected: output["build"]["nanos"] = nanos_output - if nanosp_output: + if nanosp_output and devices.nanosp.selected: output["build"]["nanosp"] = nanosp_output - if nanox_output: + if nanox_output and devices.nanox.selected: output["build"]["nanox"] = nanox_output - if stax_output: + if stax_output and devices.stax.selected: output["build"]["stax"] = stax_output log = nanos_log + nanosp_log + nanox_log + stax_log diff --git a/scripts/build_and_test/device.py b/scripts/build_and_test/device.py index 0b51ca7..e5ef5ed 100644 --- a/scripts/build_and_test/device.py +++ b/scripts/build_and_test/device.py @@ -8,8 +8,9 @@ NANOX_API_LEVEL = 5 STAX_API_LEVEL = 12 + class Device: - def __init__(self, target_name: str, model_name:str, sdk_name: str, api_level: int, enabled: bool): + def __init__(self, target_name: str, model_name: str, sdk_name: str, api_level: int, enabled: bool): self.selected = enabled self.target_name = target_name self.model_name = model_name diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 3b3e5fd..618df26 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -1,22 +1,36 @@ from pathlib import Path -from device import Devices, Device +from typing import Dict, Optional, Tuple, Union + +from build_and_test.device import Devices, Device from utils import run_cmd -def scan_variant(target: str, sdk_path: str, variant_param: str, variant_value: str, app_build_path: - Path, extra_flags: str=""): - error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) + +def scan_variant(target: str, + sdk_path: Path, + variant_param: Optional[str], + variant_value: str, + app_build_path: Path, + extra_flags: str = "") -> Tuple[int, str]: + run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) if variant_param: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} -j ENABLE_SDK_WERROR=1 scan-build", cwd=app_build_path, no_throw=True) + status, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} " + "-j ENABLE_SDK_WERROR=1 scan-build", + cwd=app_build_path, no_throw=True) else: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j ENABLE_SDK_WERROR=1 scan-build", cwd=app_build_path, no_throw=True) + status, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j ENABLE_SDK_WERROR=1 scan-build", + cwd=app_build_path, no_throw=True) - if error: + if status: print("\t=> KO") - return error, log + return status, log -def scan_all_variants(target: str, sdk_path: str, variant_param: str, variant_list: list, app_build_path: Path): +def scan_all_variants(target: str, + sdk_path: Path, + variant_param: Optional[str], + variant_list: list, + app_build_path: Path) -> Tuple[Dict, str]: output = {} error_log = "" for variant in variant_list: @@ -31,28 +45,36 @@ def scan_all_variants(target: str, sdk_path: str, variant_param: str, variant_li return output, error_log -def scan_device(device: Device, variant_param: str, app_build_path: Path, sdk_path: Path, app_json: dict): +def scan_device(device: Device, + variant_param: Optional[str], + app_build_path: Path, + sdk_path: Path, + app_json: dict) -> Tuple[Union[str, Dict[str, str]], str]: whitelist = app_json.get("scan_whitelist", "[]") error_log = "" if not device.selected: - return None, error_log + return "Unselected", error_log if device.model_name not in whitelist: - return "Skipped", error_log + return "Blacklisted", error_log variants = app_json.get(f"variants_{device.model_name}", []) - variant_output = {} + variant_output: Dict[str, str] = {} if len(variants) > 0: - variant_output, error_log = scan_all_variants(device.target_name, sdk_path, variant_param, variants, app_build_path) + variant_output, error_log = scan_all_variants(device.target_name, + sdk_path, + variant_param, + variants, + app_build_path) return variant_output, error_log def scan_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: Path): - repo_name = app_json.get("name") + repo_name = app_json["name"] variant_param = app_json.get("variant_param") - app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) + app_build_path = workdir / Path(repo_name + "/" + app_json.get("build_path", ".")) output = { "name": repo_name, @@ -67,13 +89,13 @@ def scan_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output, stax_log = scan_device(devices.stax, variant_param, app_build_path, sdk_path, app_json) - if nanos_output: + if nanos_output and devices.nanos.selected: output["scan"]["nanos"] = nanos_output - if nanosp_output: + if nanosp_output and devices.nanosp.selected: output["scan"]["nanosp"] = nanosp_output - if nanox_output: + if nanox_output and devices.nanox.selected: output["scan"]["nanox"] = nanox_output - if stax_output: + if stax_output and devices.stax.selected: output["scan"]["stax"] = stax_output log = nanos_log + nanosp_log + nanox_log + stax_log diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 56d8632..e0f00b0 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -1,11 +1,15 @@ from pathlib import Path -from device import Devices, Device -from build_app import build_variant +from typing import Optional, Tuple + +from build_and_test.device import Devices, Device +from build_and_test.build_app import build_variant from utils import run_cmd -def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str): - output = {} - error, log = run_cmd(f"pytest {app_test_path}/ --tb=short -v --device {model} {test_params}", cwd=app_build_path, no_throw=True) + +def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str) -> Tuple[str, str]: + output: str + error, log = run_cmd(f"pytest {app_test_path}/ --tb=short -v --device {model} {test_params}", + cwd=app_build_path, no_throw=True) if (error): output = "Fail" @@ -15,20 +19,27 @@ def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str return output, log -def install_dependencies(app_test_path: Path): +def install_dependencies(app_test_path: Path) -> Tuple[int, str]: error, log = run_cmd("pip install -r requirements.txt", cwd=app_test_path, no_throw=True) return error, log -def test_device(device: Device, variant_param: str, app_build_path: Path, app_test_path: Path, - sdk_path: Path, extra_flags: str, blacklist: str, test_params: str): - test_output = {} + +def test_device(device: Device, + variant_param: Optional[str], + app_build_path: Path, + app_test_path: Path, + sdk_path: Path, + extra_flags: str, + blacklist: str, + test_params: str) -> Tuple[str, str]: + test_output: str log = "" if not device.selected: - return None, log + return "Unselected", log if device.model_name in blacklist: - return "Skipped", log + return "Blacklisted", log error, log = install_dependencies(app_test_path) if error: @@ -46,10 +57,10 @@ def test_device(device: Device, variant_param: str, app_build_path: Path, app_te def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: Path): - repo_name = app_json.get("name") + repo_name = app_json["name"] variant_param = app_json.get("variant_param") - app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) - app_test_path = workdir / Path(app_json.get("name") + "/" + app_json.get("test_dir", ".")) + app_build_path = workdir / Path(repo_name + "/" + app_json.get("build_path", ".")) + app_test_path = workdir / Path(repo_name + "/" + app_json.get("test_dir", ".")) extra_flags = app_json.get("extra_flags", "") blacklist = app_json.get("build_blacklist", "[]") @@ -58,32 +69,31 @@ def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: } output["test"] = {} - blacklist = app_json.get(f"test_blacklist", []) + blacklist = app_json.get("test_blacklist", []) test_params = app_json.get("test_param_nanos", "") nanos_output, nanos_log = test_device(devices.nanos, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) + sdk_path, extra_flags, blacklist, test_params) test_params = app_json.get("test_param_nanosp", "") nanosp_output, nanosp_log = test_device(devices.nanosp, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) - + sdk_path, extra_flags, blacklist, test_params) test_params = app_json.get("test_param_nanox", "") nanox_output, nanox_log = test_device(devices.nanox, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) + sdk_path, extra_flags, blacklist, test_params) test_params = app_json.get("test_param_stax", "") stax_output, stax_log = test_device(devices.stax, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) + sdk_path, extra_flags, blacklist, test_params) - if nanos_output: + if nanos_output and devices.nanos.selected: output["test"]["nanos"] = nanos_output - if nanosp_output: + if nanosp_output and devices.nanosp.selected: output["test"]["nanosp"] = nanosp_output - if nanox_output: + if nanox_output and devices.nanox.selected: output["test"]["nanox"] = nanox_output - if stax_output: + if stax_output and devices.stax.selected: output["test"]["stax"] = stax_output print(output) diff --git a/scripts/create_app_list/main.py b/scripts/create_app_list/__init__.py old mode 100755 new mode 100644 similarity index 58% rename from scripts/create_app_list/main.py rename to scripts/create_app_list/__init__.py index 5d27a0f..15755cf --- a/scripts/create_app_list/main.py +++ b/scripts/create_app_list/__init__.py @@ -1,28 +1,13 @@ -from argparse import ArgumentParser +from argparse import Namespace from pathlib import Path import json -from parse_github import parse_github -import sys -sys.path.append('..') -from gen_variant import gen_variant +from create_app_list.parse_github import parse_github +from create_app_list.gen_variant import gen_variant from utils import git_setup, merge_json -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--access_token", required=True, type=str) - parser.add_argument("--workdir", required=False, type=str, default="workdir") - parser.add_argument("--extra_info_file", required=False, type=Path, default=Path("input_files/extra_info.json")) - parser.add_argument("--repo_file", required=False, type=Path) - parser.add_argument("--skip_setup", required=False, action='store_true') - - parser.add_argument("--full_output_file", required=False, type=Path, default=Path("output_files/full_out.json")) - parser.add_argument("--repo_output_file", required=False, type=Path, default=Path("output_files/repo.json")) - parser.add_argument("--variant_output_file", required=False, type=Path, default=Path("output_files/variant.json")) - - args = parser.parse_args() +def main(args: Namespace) -> None: abs_workdir = Path.cwd()/args.workdir if not abs_workdir.exists(): @@ -49,7 +34,7 @@ repo_name = repo.get("name") repo_ref = repo.get("ref") repo_url = repo.get("url") - repo_build_path = abs_workdir/Path(repo_name)/Path(repo.get("build_path", ".")) + repo_build_path = abs_workdir / Path(repo_name) / Path(repo.get("build_path", ".")) if not args.skip_setup: print("Cloning repo") @@ -69,4 +54,3 @@ with open(args.full_output_file, 'w') as json_file: json.dump(full_output, json_file, indent=1) - diff --git a/scripts/create_app_list/app_load_params_check.py b/scripts/create_app_list/app_load_params_check.py index f96f801..63636c2 100755 --- a/scripts/create_app_list/app_load_params_check.py +++ b/scripts/create_app_list/app_load_params_check.py @@ -72,7 +72,7 @@ def parse_listapploadparams(app_load_params_str: str) -> Dict: return dict(app_load_params) -def check_manifest(manifest: dict, database: dict) -> None: +def check_manifest(manifest: dict, database: dict) -> int: ret = 0 for variant, data in manifest["VARIANTS"].items(): @@ -99,11 +99,12 @@ def check_manifest(manifest: dict, database: dict) -> None: # Check that the params match with the one from the database for key in APP_LOAD_PARAMS_VALUE_CHECK: - app_params_ref_value = app_params_ref.get(key) - app_load_params_value = app_load_params.get(key) + app_params_ref_value = app_params_ref[key] + app_load_params_value = app_load_params[key] if key == "appName": if len(app_load_params_value) != 1: - print(f"[ERROR] Expected a single value for 'appName' ({app_load_params_value} vs {app_params_ref_value})") + print("[ERROR] Expected a single value for 'appName' " + f"({app_load_params_value} vs {app_params_ref_value})") ret = -1 continue app_load_params_value = app_load_params_value[0] @@ -112,7 +113,8 @@ def check_manifest(manifest: dict, database: dict) -> None: app_load_params_value = ["0x000"] if len(app_load_params_value) != 1: - print(f"[ERROR] Expected a single value for 'appFlags' ({app_load_params_value} vs {app_params_ref_value})") + print("[ERROR] Expected a single value for 'appFlags' " + f"({app_load_params_value} vs {app_params_ref_value})") ret = -1 continue diff --git a/scripts/create_app_list/app_load_params_utils.py b/scripts/create_app_list/app_load_params_utils.py index 0dfb391..5853479 100755 --- a/scripts/create_app_list/app_load_params_utils.py +++ b/scripts/create_app_list/app_load_params_utils.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python3 - -from pathlib import Path import json +from pathlib import Path +from typing import Dict def format_database(database: dict) -> str: @@ -20,7 +19,7 @@ def format_database(database: dict) -> str: return database_str -def load_database(database_path: Path): +def load_database(database_path: Path) -> Dict: database = {} if database_path.exists(): with open(database_path, 'r') as f: @@ -28,7 +27,6 @@ def load_database(database_path: Path): else: with open(database_path, 'w') as f: print("File created:", database_path) - database = [] return database diff --git a/scripts/create_app_list/gen_variant.py b/scripts/create_app_list/gen_variant.py index 8b0775e..1c5f6a9 100644 --- a/scripts/create_app_list/gen_variant.py +++ b/scripts/create_app_list/gen_variant.py @@ -1,21 +1,24 @@ -#!/usr/bin/env python3 - from pathlib import Path -from makefile_dump import get_app_listvariants from collections import namedtuple +from typing import Dict, List, Union + +from create_app_list.makefile_dump import get_app_listvariants Models = namedtuple('Models', ['sdk_value', 'device_name']) MODELS = [Models("$NANOS_SDK", "nanos"), - Models("$NANOX_SDK", "nanox"), - Models("$NANOSP_SDK", "nanosp"), - Models("$STAX_SDK", "stax")] + Models("$NANOX_SDK", "nanox"), + Models("$NANOSP_SDK", "nanosp"), + Models("$STAX_SDK", "stax")] -def gen_variant(app_name: str, build_path: str, output_file: Path = "", workdir: Path = "") -> dict: +def gen_variant(app_name: str, + build_path: Path, + output_file: Path = Path(), + workdir: Path = Path()) -> dict: print(f"Generating for {app_name}") - database_params = { + database_params: Dict[str, Union[str, List]] = { "name": app_name, } @@ -23,8 +26,8 @@ def gen_variant(app_name: str, build_path: str, output_file: Path = "", workdir: for model in MODELS: try: variant_param_name, variants = get_app_listvariants(build_path, model.sdk_value, allow_failure=True) - except: - print("Skipping generation due to error") + except Exception as e: + print(f"Skipping generation due to error ({e})") continue database_params["variant_param"] = variant_param_name diff --git a/scripts/create_app_list/makefile_dump.py b/scripts/create_app_list/makefile_dump.py index 41b5e0d..a1e0dbd 100755 --- a/scripts/create_app_list/makefile_dump.py +++ b/scripts/create_app_list/makefile_dump.py @@ -1,18 +1,20 @@ -#!/usr/bin/env python3 - """ This is a pure duplicate of https://github.com/LedgerHQ/ledger-app-workflows/blob/master/scripts/makefile_dump.py This is to allow easily generating the db from the apps code. """ -from utils import run_cmd from pathlib import Path from typing import Tuple, List -def get_app_listvariants(app_build_path: Path, sdk: str = "$NANOS_SDK", allow_failure: bool = False) -> Tuple[str, List[str]]: +from utils import run_cmd + + +def get_app_listvariants(app_build_path: Path, + sdk: str = "$NANOS_SDK", + allow_failure: bool = False) -> Tuple[str, List[str]]: # Using listvariants Makefile target - listvariants = run_cmd(f"make BOLOS_SDK={sdk} listvariants", cwd=app_build_path, no_throw=allow_failure) + _, listvariants = run_cmd(f"make BOLOS_SDK={sdk} listvariants", cwd=app_build_path, no_throw=allow_failure) if "VARIANTS" not in listvariants: raise ValueError(f"Invalid variants retrieved: {listvariants}") diff --git a/scripts/create_app_list/parse_github.py b/scripts/create_app_list/parse_github.py index d0035ca..a81b6bf 100755 --- a/scripts/create_app_list/parse_github.py +++ b/scripts/create_app_list/parse_github.py @@ -1,22 +1,26 @@ import requests -import json -from pathlib import Path +from dataclasses import dataclass, asdict +from typing import Dict, List base_url = "https://api.github.com" org_name = "LedgerHQ" repos_endpoint = f"{base_url}/orgs/{org_name}/repos" -params = { - "type": "public", - "archived": "false", - "sort": "full_name", - "page": 1, - "per_page": 100 -} +@dataclass +class Params: + type: str + archived: str + sort: str + page: int + per_page: int -def parse_github(access_token: str = "") -> str: + +params = Params("public", "false", "full_name", 1, 100) + + +def parse_github(access_token: str = "") -> List[Dict[str, str]]: repos = [] headers = { "Authorization": f"Bearer {access_token}", @@ -24,7 +28,7 @@ def parse_github(access_token: str = "") -> str: } while True: - response = requests.get(repos_endpoint, params=params, headers=headers) + response = requests.get(repos_endpoint, params=asdict(params), headers=headers) repos_data = response.json() if not repos_data: # No more repositories to fetch break @@ -46,6 +50,6 @@ def parse_github(access_token: str = "") -> str: owner_name = parent_data["parent"]["owner"]["login"] repos.append({"name": repo_name, "owner": owner_name, "ref": ref, "url": repo_url}) - params["page"] += 1 + params.page += 1 return repos diff --git a/scripts/create_app_list/utils.py b/scripts/create_app_list/utils.py deleted file mode 100644 index a6c7a2f..0000000 --- a/scripts/create_app_list/utils.py +++ /dev/null @@ -1,52 +0,0 @@ -import subprocess -from pathlib import Path - -def run_cmd(cmd: str, - cwd: Path, - print_output: bool = True, - no_throw: bool = False) -> str: - print(f"[run_cmd] Running: {cmd} from {cwd}") - - ret = subprocess.run(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - cwd=cwd) - if no_throw is False and ret.returncode: - print(f"[run_cmd] Error {ret.returncode} raised while running cmd: {cmd}") - print("[run_cmd] Output was:") - print(ret.stdout) - raise ValueError() - - if ret.returncode: - print(f"[run_cmd] Output:\n{ret.stdout}") - - return ret.stdout.strip() - - -def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): - # Force clone in https over SSH - GIT_CONFIG = ' -c url."https://github.com/".insteadOf="git@github.com:" -c url."https://".insteadOf="git://"' - - if not Path.exists(workdir/Path(repo_name)): - run_cmd(f"git {GIT_CONFIG} clone {repo_url} --recurse-submodules {repo_name}", cwd=workdir) - else: - run_cmd(f"git fetch", cwd=workdir/Path(repo_name)) - - run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name)) - run_cmd("git submodule update --recursive", cwd=workdir/Path(repo_name)) - - -def merge_json(json1: dict, json2: dict, key: str): - merged_data = [] - - for obj1 in json1: - merged_obj = obj1.copy() - for obj2 in json2: - if obj1[key] == obj2[key]: - merged_obj.update(obj2) - break - merged_data.append(merged_obj) - - return merged_data diff --git a/scripts/entrypoint.py b/scripts/entrypoint.py new file mode 100644 index 0000000..10039fc --- /dev/null +++ b/scripts/entrypoint.py @@ -0,0 +1,106 @@ +import sys +from argparse import ArgumentParser, Namespace +from pathlib import Path + +sys.path.insert(1, str(Path(__file__).resolve().parent)) + + +def parse_args() -> Namespace: + parser = ArgumentParser() + subparsers = parser.add_subparsers(help="Specific operation", dest="operation") + + # split_input + subparser = subparsers.add_parser('split_input') + subparser.add_argument("--split_count", required=False, type=int, default=10) + subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/input.json")) + + # build_and_test + subparser = subparsers.add_parser('build_and_test') + subparser.add_argument("--skip_setup", action='store_true') + + subparser.add_argument("--all", action='store_true') + subparser.add_argument("--nanos", action='store_true') + subparser.add_argument("--nanosp", action='store_true') + subparser.add_argument("--nanox", action='store_true') + subparser.add_argument("--stax", action='store_true') + + subparser.add_argument("--test", action='store_true') + subparser.add_argument("--build", action='store_true') + subparser.add_argument("--scan_build", action='store_true') + + subparser.add_argument("--sdk_ref", required=False, type=Path, default="origin/master") + + subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) + subparser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) + subparser.add_argument("--logs_file", required=False, type=Path, + default=Path("output_files/error_logs.txt")) + subparser.add_argument("--workdir", required=False, type=str, default="workdir") + + subparser.add_argument("--use_sha1_from_live", required=False, action='store_true') + subparser.add_argument("--provider", required=False, type=str) + subparser.add_argument("--device", required=False, type=str) + subparser.add_argument("--version", required=False, type=str) + + # output_scripts + # # convert + subparser = subparsers.add_parser('convert_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--output_file", required=False, type=Path) + subparser.add_argument("--key", required=False, type=str, default="build") + # # merge + subparser = subparsers.add_parser('merge_output', description="Merge JSON files based on a specified key") + subparser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") + subparser.add_argument("--output_file", help="Output merged JSON file") + subparser.add_argument("--key", help="Key to use for merging") + # # status + subparser = subparsers.add_parser('status_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--key", required=True) + # # slack + subparser = subparsers.add_parser('slack_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--output_file", required=False, type=Path) + subparser.add_argument("--key", required=False, type=str, default="build") + subparser.add_argument("--devices", required=False, type=str) + subparser.add_argument("--url", required=False, type=str) + + # create_app_list + subparser = subparsers.add_parser('create_app_list') + subparser.add_argument("--access_token", required=True, type=str) + subparser.add_argument("--workdir", required=False, type=str, default="workdir") + subparser.add_argument("--extra_info_file", required=False, type=Path, default=Path("input_files/extra_info.json")) + subparser.add_argument("--repo_file", required=False, type=Path) + subparser.add_argument("--skip_setup", required=False, action='store_true') + + subparser.add_argument("--full_output_file", required=False, type=Path, default=Path("output_files/full_out.json")) + subparser.add_argument("--repo_output_file", required=False, type=Path, default=Path("output_files/repo.json")) + subparser.add_argument("--variant_output_file", required=False, type=Path, + default=Path("output_files/variant.json")) + + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + if args.operation == 'split_input': + import split_input + split_input.main(args) + elif args.operation == 'build_and_test': + import build_and_test + build_and_test.main(args) + elif args.operation == 'convert_output': + import output_scripts.convert + output_scripts.convert.main(args) + elif args.operation == 'merge_output': + import output_scripts.merge + output_scripts.merge.main(args) + elif args.operation == 'status_output': + import output_scripts.status + output_scripts.status.main(args) + elif args.operation == 'slack_output': + import output_scripts.slack + output_scripts.slack.main(args) + elif args.operation == 'create_app_list': + import create_app_list + create_app_list.main(args) diff --git a/scripts/output_scripts/__init__.py b/scripts/output_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/output_scripts/convert.py b/scripts/output_scripts/convert.py old mode 100755 new mode 100644 index 7ebcce8..d13dcaf --- a/scripts/output_scripts/convert.py +++ b/scripts/output_scripts/convert.py @@ -1,6 +1,6 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace + def json_to_markdown(json_list, key): # Set up the markdown table headers @@ -20,9 +20,15 @@ def json_to_markdown(json_list, key): for d in ["nanos", "nanosp", "nanox", "stax"]: build_data = app_data.get(d, {}) if isinstance(build_data, dict): # nested structure - status_icon = ":red_circle:" if "Fail" in build_data.values() else ":heavy_check_mark:" if "Success" in build_data.values() else ":fast_forward:" if "Skipped" in build_data.values() else "" - else: - status_icon = ":heavy_check_mark:" if build_data == "Success" else ":red_circle:" if build_data == "Fail" else ":fast_forward:" if build_data == "Skipped" else "" + status_icon = (":red_circle:" if "Fail" in build_data.values() + else ":heavy_check_mark:" if "Success" in build_data.values() + else ":fast_forward:" if "Blacklisted" in build_data.values() + else "") + else: + status_icon = (":heavy_check_mark:" if build_data == "Success" + else ":red_circle:" if build_data == "Fail" + else ":fast_forward:" if build_data == "Blacklisted" + else "") row += " {} |".format(status_icon) markdown_table += row + "\n" @@ -62,15 +68,7 @@ def count_status(json_list, key): """ -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--output_file", required=False, type=Path) - parser.add_argument("--key", required=False, type=str, default="build") - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_list = json.load(json_file) diff --git a/scripts/output_scripts/merge.py b/scripts/output_scripts/merge.py index b1688ef..083721e 100644 --- a/scripts/output_scripts/merge.py +++ b/scripts/output_scripts/merge.py @@ -1,11 +1,12 @@ -import argparse import glob import json +from argparse import Namespace + def merge_jsons(json1, json2, key): # Create a dictionary to store the merged data by name merged_data = {} - + # Merge data from the first JSON array for item in json1: name = item["name"] @@ -13,7 +14,7 @@ def merge_jsons(json1, json2, key): merged_data[name] = item[key] else: merged_data[name].update(item[key]) - + # Merge data from the second JSON array for item in json2: name = item["name"] @@ -21,12 +22,13 @@ def merge_jsons(json1, json2, key): merged_data[name] = item[key] else: merged_data[name].update(item[key]) - + # Convert the merged dictionary back to a list of JSON objects merged_json = [{"name": name, key: merged_data[name]} for name in merged_data] - + return merged_json + def merge_multiple_jsons(input_files, key): result = {} for j in input_files: @@ -36,14 +38,7 @@ def merge_multiple_jsons(input_files, key): return result -def main(): - parser = argparse.ArgumentParser(description="Merge JSON files based on a specified key") - parser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") - parser.add_argument("--output_file", help="Output merged JSON file") - parser.add_argument("--key", help="Key to use for merging") - - args = parser.parse_args() - +def main(args: Namespace) -> None: input_files = glob.glob(args.input_pattern) if not input_files: print("No input files found.") @@ -52,7 +47,3 @@ def main(): merged_json = merge_multiple_jsons(input_files, args.key) with open(args.output_file, 'w') as f: json.dump(merged_json, f, indent=1) - - -if __name__ == "__main__": - main() diff --git a/scripts/output_scripts/slack.py b/scripts/output_scripts/slack.py index cea4bd1..7e309a1 100644 --- a/scripts/output_scripts/slack.py +++ b/scripts/output_scripts/slack.py @@ -1,6 +1,6 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace + def count_test_status(json_list): success_count = 0 @@ -32,6 +32,7 @@ def count_test_status(json_list): return success_count, fail_count, total_count, fail_list + def count_status(json_list, key): success_count = 0 fail_count = 0 @@ -64,18 +65,7 @@ def count_status(json_list, key): return success_count, fail_count, total_count, fail_list -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--output_file", required=False, type=Path) - parser.add_argument("--key", required=False, type=str, default="build") - parser.add_argument("--devices", required=False, type=str) - parser.add_argument("--url", required=False, type=str) - - args = parser.parse_args() - - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_list = json.load(json_file) @@ -84,13 +74,11 @@ def count_status(json_list, key): else: success_count, fail_count, total_count, fail_list = count_status(json_list, args.key) - title = f"{args.key}" if args.devices: title += f" on {args.devices}" - status_detail = "" if fail_count == 0: status = f":large_green_circle: Success for {total_count} apps" @@ -102,9 +90,9 @@ def count_status(json_list, key): fail_status += f"\t• {app_name}\n" if isinstance(details, dict): for device, variant in details.items(): - fail_status += f"\t\t - {device}: \n"; + fail_status += f"\t\t - {device}: \n" for v in variant: - fail_status += f"\t\t\t{v}\n"; + fail_status += f"\t\t\t{v}\n" else: for device in details: fail_status += f"\t\t- {device}\n" @@ -121,4 +109,3 @@ def count_status(json_list, key): if args.output_file: with open(args.output_file, 'w') as f: json.dump(slack_json, f, indent=1) - diff --git a/scripts/output_scripts/status.py b/scripts/output_scripts/status.py old mode 100755 new mode 100644 index bc3defd..3c15f20 --- a/scripts/output_scripts/status.py +++ b/scripts/output_scripts/status.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def check_status(json_data, key): @@ -19,15 +18,7 @@ def check_status(json_data, key): raise ValueError(f"Failed for {app_name}") -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--key", required=True) - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_data = json.load(json_file) - check_status(json_data, args.key) diff --git a/scripts/split_input/split_input.py b/scripts/split_input.py old mode 100755 new mode 100644 similarity index 70% rename from scripts/split_input/split_input.py rename to scripts/split_input.py index 70a6dff..4559cc3 --- a/scripts/split_input/split_input.py +++ b/scripts/split_input.py @@ -1,16 +1,10 @@ import json from pathlib import Path -from argparse import ArgumentParser +from argparse import Namespace -if __name__ == "__main__": - parser = ArgumentParser() - input_json = {} - - parser.add_argument("--input_file", required=False, type=Path, - default=Path("input_files/input.json")) - parser.add_argument("--split_count", required=False, type=Path, default=10) - args = parser.parse_args() +def main(args: Namespace) -> None: + input_json = {} if Path(args.input_file).exists(): with open(args.input_file) as json_file: @@ -22,7 +16,6 @@ num_files = args.split_count items_per_file = len(input_json) // num_files - for file_num in range(num_files): start_idx = file_num * items_per_file @@ -32,5 +25,5 @@ with open(file_name, 'w') as file: json.dump(input_json[start_idx:end_idx], file, indent=1) # Split data into ten JSON files - + print(f"Data split into {args.split_count} JSON files.") diff --git a/scripts/build_and_test/utils.py b/scripts/utils.py similarity index 75% rename from scripts/build_and_test/utils.py rename to scripts/utils.py index 9d16ef9..896c940 100644 --- a/scripts/build_and_test/utils.py +++ b/scripts/utils.py @@ -1,13 +1,14 @@ import subprocess import shutil from pathlib import Path +from typing import List, Tuple def run_cmd(cmd: str, cwd: Path, print_output: bool = True, - no_throw: bool = False) -> str: - error_log = "" + no_throw: bool = False) -> Tuple[int, str]: + stdout = "" print(f"[run_cmd] Running: {cmd} from {cwd}") ret = subprocess.run(cmd, @@ -15,32 +16,30 @@ def run_cmd(cmd: str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, - cwd=cwd) - if no_throw is False and ret.returncode: - print(f"[run_cmd] Error {ret.returncode} raised while running cmd: {cmd}") - print("[run_cmd] Output was:") - print(ret.stdout) - raise ValueError() + cwd=cwd, + check=not no_throw) if ret.returncode: print(f"[run_cmd] Output:\n{ret.stdout}") - - error_log = f''' + stdout = f''' ############################################################################### [run_cmd] Running: {cmd} from {cwd}" ############################################################################### ''' + ret.stdout - return ret.returncode, error_log + else: + stdout = ret.stdout.strip() + + return ret.returncode, stdout -def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): +def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path) -> None: # Force clone in https over SSH GIT_CONFIG = ' -c url."https://github.com/".insteadOf="git@github.com:" -c url."https://".insteadOf="git://"' if not Path.exists(workdir/Path(repo_name)): run_cmd(f"git {GIT_CONFIG} clone {repo_url} {repo_name}", cwd=workdir) else: - run_cmd(f"git fetch", cwd=workdir/Path(repo_name)) + run_cmd("git fetch", cwd=workdir/Path(repo_name)) error, _ = run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name), no_throw=True) if error: @@ -48,14 +47,16 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): shutil.rmtree(workdir/Path(repo_name)) return - error, _ = run_cmd(f"git {GIT_CONFIG} submodule update --init --recursive", cwd=workdir/Path(repo_name), no_throw=True) + error, _ = run_cmd(f"git {GIT_CONFIG} submodule update --init --recursive", + cwd=workdir/Path(repo_name), + no_throw=True) if error: print("Error: removing folder") shutil.rmtree(workdir/Path(repo_name)) return -def merge_json(json1: dict, json2: dict, key: str): +def merge_json(json1: list, json2: list, key: str) -> List: merged_data = [] for obj1 in json1: