diff --git a/.github/workflows/_build_devices.yml b/.github/workflows/_build_devices.yml index 6e58d8f..2676b7b 100644 --- a/.github/workflows/_build_devices.yml +++ b/.github/workflows/_build_devices.yml @@ -25,33 +25,6 @@ on: required: false default: true - pull_request: - push: - inputs: - sdk_ref: - type: string - required: false - default: 'master' - - run_nanos: - type: boolean - required: false - default: true - run_nanosp: - type: boolean - required: false - default: true - run_nanox: - type: boolean - required: false - default: true - run_stax: - type: boolean - required: false - default: true - - - jobs: build-build: name: Build for all targets @@ -70,7 +43,7 @@ jobs: - name: Launch build nanos if: ${{ inputs.run_nanos }} run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref}} --input_file input_files/test_input.json --build --nanos --skip_setup --output_file build_input.json + python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref}} --input_file input_files/test_input.json --build --nanos --skip_setup --output_file build_nanos.json - name: Launch build nanosp if: ${{ inputs.run_nanosp }} @@ -89,7 +62,7 @@ jobs: - name: Merge output files run: | - python3 scripts/output_scripts/merge.py --input_files build_nanos.json build_nanosp.json build_nanox.json build_stax.json --output_file build_output.json --merge_key "name" + python3 scripts/output_scripts/merge.py --input_pattern "build_*.json" --output_file build_output.json --merge_key "name" #- name: Push info to DB # run: | diff --git a/.github/workflows/_test_devices.yml b/.github/workflows/_test_devices.yml index 08d2e02..f023e5a 100644 --- a/.github/workflows/_test_devices.yml +++ b/.github/workflows/_test_devices.yml @@ -29,36 +29,6 @@ on: type: boolean required: false default: true - - pull_request: - push: - inputs: - sdk_ref_nanosp: - type: string - required: false - default: 'API_LEVEL_1' - run_nanosp: - type: boolean - required: false - default: true - - sdk_ref_nanox: - type: string - required: false - default: 'API_LEVEL_5' - run_nanox: - type: boolean - required: false - default: true - - sdk_ref_stax: - type: string - required: false - default: 'API_LEVEL_12' - run_stax: - type: boolean - required: false - default: true jobs: test: @@ -78,21 +48,21 @@ jobs: - name: Launch test nanosp if: ${{ inputs.run_nanosp }} run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref_nanosp }} --input_file input_files/test.json --test --nanosp --skip_setup --output_file test_nanosp.json + python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref_nanosp }} --input_file input_files/test_input.json --test --nanosp --skip_setup --output_file test_nanosp.json - name: Launch test nanox if: ${{ inputs.run_nanox }} run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref_nanox }} --input_file input_files/test.json --test --nanox --skip_setup --output_file test_nanox.json + python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref_nanox }} --input_file input_files/test_input.json --test --nanox --skip_setup --output_file test_nanox.json - name: Launch test stax if: ${{ inputs.run_stax }} run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref_stax }} --input_file input_files/test.json --test --stax --skip_setup --output_file test_stax.json + python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref_stax }} --input_file input_files/test_input.json --test --stax --skip_setup --output_file test_stax.json - name: Merge output files run: | - python3 scripts/output_scripts/merge.py --input_files test_nanosp.json test_nanox.json test_stax.json --output_file test_output.json --merge_key "name" + python3 scripts/output_scripts/merge.py --input_pattern "test_*.json" --output_file test_output.json --merge_key "name" - name: Archive output file uses: actions/upload-artifact@v3 diff --git a/.github/workflows/check_outdated_build_db.yaml b/.github/workflows/check_outdated_build_db.yaml index dd4d561..6e414ba 100644 --- a/.github/workflows/check_outdated_build_db.yaml +++ b/.github/workflows/check_outdated_build_db.yaml @@ -2,7 +2,6 @@ name: Check outdated build db on: workflow_dispatch: - #push: jobs: compare_files: diff --git a/input_files/test_input.json b/input_files/test_input.json index d48af4d..ffd2635 100644 --- a/input_files/test_input.json +++ b/input_files/test_input.json @@ -16,226 +16,5 @@ "variants_nanosp": [ "ACA" ] - }, - { - "name": "app-aergo", - "owner": "aergoio", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-aergo", - "build_path": "workdir/app-near", - "test_dir": "tests/functional", - "extra_flags": "TESTING=1 POPULATE=1" - }, - { - "name": "app-aeternity", - "owner": "LedgerHQ", - "ref": "master", - "url": "https://github.com/LedgerHQ/app-aeternity", - "variant_param": "COIN", - "variants_nanos": [ - "aeternity" - ], - "variants_nanox": [ - "aeternity" - ], - "variants_nanosp": [ - "aeternity" - ], - "variants_stax": [ - "aeternity" - ] - }, - { - "name": "app-age", - "owner": "LedgerHQ", - "ref": "main", - "url": "https://github.com/LedgerHQ/app-age" - }, - { - "name": "app-aion", - "owner": "aionnetwork", - "ref": "master", - "url": "https://github.com/LedgerHQ/app-aion", - "variant_param": "CHAIN", - "variants_nanos": [ - "aion" - ], - "variants_nanox": [ - "aion" - ], - "variants_nanosp": [ - "aion" - ], - "variants_stax": [ - "aion" - ] - }, - { - "name": "app-alephzero", - "owner": "Zondax", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-alephzero", - "build_path": "app", - "variant_param": "COIN", - "variants_nanos": [ - "AZERO", - "AZERO_XL" - ], - "variants_nanox": [ - "AZERO" - ], - "variants_nanosp": [ - "AZERO" - ], - "variants_stax": [ - "AZERO" - ] - }, - { - "name": "app-algorand", - "owner": "algorand", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-algorand", - "build_path": "app", - "variant_param": "COIN", - "variants_nanos": [ - "ALGO" - ], - "variants_nanox": [ - "ALGO" - ], - "variants_nanosp": [ - "ALGO" - ], - "variants_stax": [ - "ALGO" - ] - }, - { - "name": "app-aptos", - "owner": "pontem-network", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-aptos", - "variant_param": "COIN", - "variants_nanos": [ - "APTOS" - ], - "variants_nanox": [ - "APTOS" - ], - "variants_nanosp": [ - "APTOS" - ], - "variants_stax": [ - "APTOS" - ] - }, - { - "name": "app-ardor", - "owner": "jelurida-dev", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-ardor", - "variant_param": "COIN", - "variants_nanos": [ - "ardor" - ], - "variants_nanox": [ - "ardor" - ], - "variants_nanosp": [ - "ardor" - ], - "variants_stax": [ - "ardor" - ] - }, - { - "name": "app-ark", - "owner": "ArkEcosystem", - "ref": "master", - "url": "https://github.com/LedgerHQ/app-ark", - "variant_param": "COIN", - "variants_nanos": [ - "ark" - ], - "variants_nanox": [ - "ark" - ], - "variants_nanosp": [ - "ark" - ], - "variants_stax": [ - "ark" - ] - }, - { - "name": "app-arweave", - "owner": "Zondax", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-arweave", - "build_path": "app", - "variant_param": "COIN", - "variants_nanos": [ - "ARWEAVE" - ], - "variants_nanox": [ - "ARWEAVE" - ], - "variants_nanosp": [ - "ARWEAVE" - ], - "variants_stax": [ - "ARWEAVE" - ] - }, - { - "name": "app-astar", - "owner": "Zondax", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-astar", - "build_path": "app", - "variant_param": "COIN", - "variants_nanos": [ - "ASTR", - "ASTR_XL" - ], - "variants_nanox": [ - "ASTR" - ], - "variants_nanosp": [ - "ASTR" - ], - "variants_stax": [ - "ASTR" - ] - }, - { - "name": "app-avalanche", - "owner": "ava-labs", - "ref": "develop", - "url": "https://github.com/LedgerHQ/app-avalanche", - "build_blacklist": [ - "nanos", - "nanosp", - "nanox", - "stax" - ], - "test_blacklist": [ - "nanos", - "nanosp", - "nanox", - "stax" - ], - "build_path": "app", - "variant_param": "COIN", - "variants_nanos": [ - "AVAX" - ], - "variants_nanox": [ - "AVAX" - ], - "variants_nanosp": [ - "AVAX" - ] } ] \ No newline at end of file diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 9d97232..e683287 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -49,9 +49,10 @@ def build_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) blacklist = app_json.get("build_blacklist", "[]") - output = {} - output[repo_name] = {} - output[repo_name]["build"] = {} + output = { + "name": repo_name, + } + output["build"] = {} if "nanos" in blacklist: nanos_output = "Skipped" @@ -74,12 +75,12 @@ def build_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output = build_device(devices.stax, variant_param, app_build_path, sdk_path, app_json) if nanos_output: - output[repo_name]["build"]["nanos"] = nanos_output + output["build"]["nanos"] = nanos_output if nanosp_output: - output[repo_name]["build"]["nanosp"] = nanosp_output + output["build"]["nanosp"] = nanosp_output if nanox_output: - output[repo_name]["build"]["nanox"] = nanox_output + output["build"]["nanox"] = nanox_output if stax_output: - output[repo_name]["build"]["stax"] = stax_output + output["build"]["stax"] = stax_output return output diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 1bcb25e..0954225 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -42,17 +42,18 @@ def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: extra_flags = app_json.get("extra_flags", "") blacklist = app_json.get("build_blacklist", "[]") - output = {} - output[repo_name] = {} - output[repo_name]["test"] = {} + output = { + "name": repo_name, + } + output["test"] = {} # Install dependencies first if install_dependencies(app_test_path): print("Error installing dependencies") - output[repo_name]["test"]["nanos"] = "Fail" - output[repo_name]["test"]["nanosp"] = "Fail" - output[repo_name]["test"]["nanox"] = "Fail" - output[repo_name]["test"]["stax"] = "Fail" + output["test"]["nanos"] = "Fail" + output["test"]["nanosp"] = "Fail" + output["test"]["nanox"] = "Fail" + output["test"]["stax"] = "Fail" return output blacklist = app_json.get(f"test_blacklist", []) @@ -79,13 +80,13 @@ def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output = test_device(devices.stax, variant_param, app_build_path, app_test_path, sdk_path, extra_flags, app_json) if nanos_output: - output[repo_name]["test"]["nanos"] = nanos_output + output["test"]["nanos"] = nanos_output if nanosp_output: - output[repo_name]["test"]["nanosp"] = nanosp_output + output["test"]["nanosp"] = nanosp_output if nanox_output: - output[repo_name]["test"]["nanox"] = nanox_output + output["test"]["nanox"] = nanox_output if stax_output: - output[repo_name]["test"]["stax"] = stax_output + output["test"]["stax"] = stax_output print(output) diff --git a/scripts/output_scripts/convert.py b/scripts/output_scripts/convert.py index d702b0d..638831e 100755 --- a/scripts/output_scripts/convert.py +++ b/scripts/output_scripts/convert.py @@ -3,18 +3,25 @@ from pathlib import Path def json_to_markdown(json_data): - markdown_table = "| App Name | build_nanos | build_nanosp | build_nanox | build_stax |\n" - markdown_table += "|--------------|-------------|--------------|-------------|------------|\n" + # Set up the markdown table headers + markdown_table = "| App Name | build_nanos | build_nanosp | build_nanox | build_stax |\n" + markdown_table += "|----------|-------------|--------------|-------------|------------|\n" - for app_name, app_data in json_data.items(): - row = "| {} |".format(app_name) - for build_name, build_data in app_data.items(): - build_status = ", ".join("{}: {}".format(key, ":heavy_check_mark:" if value == "Success" else ":red_circle:") for key, value in build_data.items()) - row += " {} |".format(build_status) - markdown_table += row + "\n" + # Extract app_name and app_data from the json_data + app_name = json_data["name"] + app_data = json_data["test"] - return markdown_table + # Start constructing the row with app_name + row = "| {} |".format(app_name) + + # Iterate over each build and append the status + for build in ["nanos", "nanosp", "nanox", "stax"]: + build_status = app_data.get(build, None) + status_icon = ":heavy_check_mark:" if build_status == "Success" else ":red_circle:" if build_status == "Fail" else "" + row += " {} |".format(status_icon) + markdown_table += row + "\n" + return markdown_table def count_status(json_data): counts = { @@ -23,14 +30,15 @@ def count_status(json_data): "Total": 0 } - for app_data in json_data.values(): - for build_data in app_data.values(): - for status in build_data.values(): - counts["Total"] += 1 - if status == "Success": - counts["Success"] += 1 - elif status == "Fail": - counts["Fail"] += 1 + # Extract the app_data from json_data + app_data = json_data["test"] + + for status in app_data.values(): + counts["Total"] += 1 + if status == "Success": + counts["Success"] += 1 + elif status == "Fail": + counts["Fail"] += 1 return f""" Success: {counts['Success']} diff --git a/scripts/output_scripts/merge.py b/scripts/output_scripts/merge.py index 8c321cf..d441f28 100644 --- a/scripts/output_scripts/merge.py +++ b/scripts/output_scripts/merge.py @@ -1,29 +1,41 @@ import argparse +import glob import json -def merge_json_files(input_files, output_file, merge_key): - merged_data = {} +def merge_dicts(d1, d2): + for k, v in d2.items(): + if k in d1 and isinstance(d1[k], dict) and isinstance(v, dict): + merge_dicts(d1[k], v) + else: + d1[k] = v + return d1 - for input_file in input_files: - with open(input_file, 'r') as f: +def merge_multiple_jsons(input_files): + result = {} + for j in input_files: + with open(j, 'r') as f: data = json.load(f) - key_value = data.get(merge_key) - if key_value is not None: - merged_data[key_value] = data + merge_dicts(result, data) + return result - with open(output_file, 'w') as f: - json.dump(merged_data, f, indent=2) def main(): parser = argparse.ArgumentParser(description="Merge JSON files based on a specified key") - parser.add_argument("--input_files", nargs="+", help="List of input JSON files") + parser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") parser.add_argument("--output_file", help="Output merged JSON file") parser.add_argument("--merge_key", help="Key to use for merging") args = parser.parse_args() - merge_json_files(args.input_files, args.output_file, args.merge_key) - print(f"Merged data saved to {args.output_file}") + input_files = glob.glob(args.input_pattern) + if not input_files: + print("No input files found.") + return + + merged_json = merge_multiple_jsons(input_files) + with open(args.output_file, 'w') as f: + json.dump(merged_json, f, indent=1) + if __name__ == "__main__": main()