Skip to content

Commit

Permalink
[clean] Single script entrypoint
Browse files Browse the repository at this point in the history
  • Loading branch information
lpascal-ledger committed Nov 21, 2023
1 parent f9b9ffa commit e0ebf97
Show file tree
Hide file tree
Showing 21 changed files with 285 additions and 283 deletions.
30 changes: 15 additions & 15 deletions .github/workflows/build_all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ jobs:
uses: actions/checkout@v3

- name: Split input into 10 files
run: |
python3 scripts/split_input/split_input.py --input_file input_files/input.json
run: |
python3 scripts/entrypoint.py split_input --input_file input_files/input.json
- name: Archive output file
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -186,17 +186,17 @@ jobs:

- name: Setup repos
if: ${{ inputs.use_sha1_from_live }}
run: |
python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }}
run: |
python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }}
- name: Setup repos
if: ${{ !inputs.use_sha1_from_live }}
run: |
python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json
run: |
python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json
- name: Launch build
run: |
python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
run: |
python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
#- name: Push info to DB
# run: |
Expand Down Expand Up @@ -534,8 +534,8 @@ jobs:
name: build_${{ matrix.device }}_10.json

- name: Merge output files
run: |
python3 scripts/output_scripts/merge.py --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build"
run: |
python3 scripts/entrypoint.py merge_output --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build"
- name: Archive output file
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -577,20 +577,20 @@ jobs:
name: merged_build_stax.json

- name: Merge output files
run: |
python3 scripts/output_scripts/merge.py --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build"
run: |
python3 scripts/entrypoint.py merge_output --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build"
- name: Convert to markdown
run: |
python3 scripts/output_scripts/convert.py --input_file full_build_output.json --output_file out.md --key build
python3 scripts/entrypoint.py convert_output --input_file full_build_output.json --output_file out.md --key build
cat out.md >> $GITHUB_STEP_SUMMARY
- name: Echo GHA url
run: echo "url"=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_ENV

- name: Convert to slack json
run: |
python3 scripts/output_scripts/slack.py --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
python3 scripts/entrypoint.py slack_output --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
- name: Send custom JSON data to Slack workflow
if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }}
Expand All @@ -603,7 +603,7 @@ jobs:

- name: Set job status
run: |
python3 scripts/output_scripts/status.py --input_file full_build_output.json --key build
python3 scripts/entrypoint.py status_output --input_file full_build_output.json --key build
- name: Archive output file
if: always()
Expand Down
16 changes: 2 additions & 14 deletions .github/workflows/fast-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,38 +12,26 @@ jobs:
lint:
strategy:
fail-fast: false
matrix:
directory:
- build_and_test
- create_app_list
- output_scripts
- split_input
name: Linting
runs-on: ubuntu-latest
steps:
- name: Clone
uses: actions/checkout@v3
- run: pip install flake8
- name: Flake8 lint Python code
run: find scripts/${{ matrix.directory }} -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+'
run: find scripts/ -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+'

mypy:
strategy:
fail-fast: false
matrix:
directory:
- build_and_test
- create_app_list
- output_scripts
- split_input
name: Type checking
runs-on: ubuntu-latest
steps:
- name: Clone
uses: actions/checkout@v3
- run: pip install mypy types-requests
- name: Mypy type checking
run: mypy scripts/${{ matrix.directory }}
run: mypy scripts/

misspell:
strategy:
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/refresh_inputs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ jobs:
runs-on: ubuntu-latest
container:
image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest

steps:
- name: Checkout repository
uses: actions/checkout@v3

- name: Build db
run: |
python3 scripts/create_app_list/main.py --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }}
run: |
python3 scripts/entrypoint.py create_app_list --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }}
- name: Upload file
uses: actions/upload-artifact@v3
with:
Expand Down
26 changes: 13 additions & 13 deletions .github/workflows/scan_all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ jobs:
uses: actions/checkout@v3

- name: Split input into 10 files
run: |
python3 scripts/split_input/split_input.py --input_file input_files/input.json
run: |
python3 scripts/entrypoint.py split_input --input_file input_files/input.json
- name: Archive output file
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -168,12 +168,12 @@ jobs:
name: input_${{ matrix.index }}.json

- name: Setup repos
run: |
python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json
run: |
python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json
- name: Launch scan
run: |
python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
run: |
python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
#- name: Push info to DB
# run: |
Expand Down Expand Up @@ -254,8 +254,8 @@ jobs:
name: scan_${{ matrix.device }}_10.json

- name: Merge output files
run: |
python3 scripts/output_scripts/merge.py --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan"
run: |
python3 scripts/entrypoint.py merge_output --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan"
- name: Archive output file
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -554,20 +554,20 @@ jobs:
name: merged_scan_stax.json

- name: Merge output files
run: |
python3 scripts/output_scripts/merge.py --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan"
run: |
python3 scripts/entrypoint.py merge_output --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan"
- name: Convert to markdown
run: |
python3 scripts/output_scripts/convert.py --input_file full_scan_output.json --output_file out.md --key scan
python3 scripts/entrypoint.py convert_output --input_file full_scan_output.json --output_file out.md --key scan
cat out.md >> $GITHUB_STEP_SUMMARY
- name: Echo GHA url
run: echo "url"=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_ENV

- name: Convert to slack json
run: |
python3 scripts/output_scripts/slack.py --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
python3 scripts/entrypoint.py slack_output --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
- name: Send custom JSON data to Slack workflow
if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }}
Expand All @@ -580,7 +580,7 @@ jobs:

- name: Set job status
run: |
python3 scripts/output_scripts/status.py --input_file full_scan_output.json --key scan
python3 scripts/entrypoint.py status_output --input_file full_scan_output.json --key scan
- name: Archive output file
uses: actions/upload-artifact@v3
Expand Down
24 changes: 12 additions & 12 deletions .github/workflows/test_all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ jobs:
uses: actions/checkout@v3

- name: Split input into 10 files
run: |
python3 scripts/split_input/split_input.py --input_file input_files/input.json
run: |
python3 scripts/entrypoint.py split_input --input_file input_files/input.json
- name: Archive output file
uses: actions/upload-artifact@v3
Expand Down Expand Up @@ -136,8 +136,8 @@ jobs:
name: input_${{ matrix.index }}.json

- name: Setup repos
run: |
python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json
run: |
python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json
- name: Install tests dependencies
run: |
Expand Down Expand Up @@ -167,22 +167,22 @@ jobs:
- name: Launch test nanosp
if: ${{ env.test_nanosp }}
run: |
python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt
run: |
python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt
- name: Launch test nanox
if: ${{ env.test_nanox }}
run: |
python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt
run: |
python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt
- name: Launch test stax
if: ${{ env.test_stax }}
run: |
python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt
run: |
python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt
- name: Merge output files
run: |
python scripts/output_scripts/merge.py --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test"
run: |
python scripts/entrypoint.py merge_output --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test"
- name: Archive output file
uses: actions/upload-artifact@v3
with:
Expand Down
104 changes: 104 additions & 0 deletions scripts/build_and_test/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import json
from pathlib import Path
from argparse import Namespace

from build_and_test.sha1 import override_sha1
from build_and_test.build_app import build_all_devices
from build_and_test.test_app import test_all_devices
from build_and_test.scan_app import scan_all_devices
from build_and_test.device import Devices
from build_and_test.utils import git_setup, merge_json

SDK_NAME = "sdk"
SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git"
SDK_BRANCH = "origin/master"


def main(args: Namespace) -> None:
input_json = {}

abs_workdir = Path.cwd() / args.workdir

if not abs_workdir.exists():
abs_workdir.mkdir()

nanos_enable = False
nanosp_enable = False
nanox_enable = False
stax_enable = False

if args.all or args.nanos:
print("Nanos enabled")
nanos_enable = True
if args.all or args.nanosp:
print("Nanosp enabled")
nanosp_enable = True
if args.all or args.nanox:
print("Nanox enabled")
nanox_enable = True
if args.all or args.stax:
print("Stax enabled")
stax_enable = True

devices = Devices(nanos_enable, nanosp_enable, nanox_enable, stax_enable)

if Path(args.input_file).exists():
with open(args.input_file) as json_file:
input_json = json.load(json_file)
else:
print("Error: input file does not exist")
exit()

if args.use_sha1_from_live:
if not args.provider:
print("Error: you must specify provider")
exit()
if not args.device:
print("Error: you must specify device")
exit()
if not args.version:
print("Error: you must specify version")
exit()

input_json = override_sha1(input_json, args.provider, args.device, args.version)

git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir)

output = {}
test_output = {}
build_output = []
logs = ""

for app_json in input_json:
repo_name = app_json.get("name")
if not args.skip_setup:
repo_ref = app_json.get("ref")
repo_url = app_json.get("url")
print(f"Setup {repo_name}")
git_setup(repo_name, repo_ref, repo_url, abs_workdir)

if args.build:
print(f"Build {repo_name}")
build_app, log = build_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir)
build_output.append(build_app)
logs += log

if args.test:
print(f"Test {repo_name}")
test_app, log = test_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir)
build_output.append(test_app)
logs += log

if args.scan_build:
print(f"Scan build {repo_name}")
scan_app, log = scan_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir)
build_output.append(scan_app)
logs += log

output = merge_json(build_output, test_output, "name")

with open(args.output_file, 'w') as json_file:
json.dump(output, json_file, indent=1)

with open(args.logs_file, 'w') as file:
file.write(logs)
7 changes: 4 additions & 3 deletions scripts/build_and_test/build_app.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
from pathlib import Path
from device import Devices, Device
from utils import run_cmd
import os
from pathlib import Path

from build_and_test.device import Devices, Device
from build_and_test.utils import run_cmd


def build_variant(target: str,
Expand Down
Loading

0 comments on commit e0ebf97

Please sign in to comment.