From 3271a3718fd046a24fc880fbe0d6aa7229eecc73 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Mon, 25 Nov 2024 10:20:47 -0800 Subject: [PATCH] ci: update actions and images Updates some actions to run build steps in container instead of the whole workflow. --- .github/workflows/checks.yml | 41 +++--- .github/workflows/clang-tidy.yml | 16 ++- .github/workflows/compile_macos.yml | 8 +- .../ekf_functional_change_indicator.yml | 23 +-- .../workflows/ekf_update_change_indicator.yml | 32 +++-- .github/workflows/failsafe_sim.yml | 40 +++--- .github/workflows/mavros_mission_tests.yml | 133 ++---------------- .github/workflows/mavros_offboard_tests.yml | 125 ++-------------- .github/workflows/nuttx_env_config.yml | 9 +- .github/workflows/python_checks.yml | 8 +- ROMFS/px4fmu_common/init.d/rcS | 4 +- Tools/setup/ubuntu.sh | 1 + 12 files changed, 137 insertions(+), 303 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a2d7aa5f97a8..1e8d2238a1a4 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -25,28 +25,27 @@ jobs: "NO_NINJA_BUILD=1 px4_fmu-v5_default", "NO_NINJA_BUILD=1 px4_sitl_default", "px4_sitl_allyes", - "airframe_metadata", "module_documentation", - "parameters_metadata", ] - container: - image: px4io/px4-dev-nuttx-focal:2022-08-12 - options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 - - name: check environment - run: | - export - ulimit -a - - name: ${{matrix.check}} - run: make ${{matrix.check}} - - name: upload coverage - if: contains(matrix.check, 'coverage') - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: unittests - file: coverage/lcov.info + - name: Building [${{ matrix.check }}] + uses: addnab/docker-run-action@v3 + with: + image: px4io/px4-dev-nuttx-focal:2022-08-12 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + git config --global --add safe.directory /workspace + make ${{ matrix.check }} + + - name: Uploading Coverage to Codecov.io + if: contains(matrix.check, 'coverage') + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: unittests + file: coverage/lcov.info diff --git a/.github/workflows/clang-tidy.yml b/.github/workflows/clang-tidy.yml index 2736767e0bb7..89c1ffe1ec37 100644 --- a/.github/workflows/clang-tidy.yml +++ b/.github/workflows/clang-tidy.yml @@ -11,13 +11,17 @@ on: jobs: build: runs-on: ubuntu-latest - container: px4io/px4-dev-clang:2021-09-08 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 with: - token: ${{secrets.ACCESS_TOKEN}} + fetch-depth: 0 - - uses: corrupt952/actions-retry-command@v1.0.7 + - name: Testing (clang-tidy-quiet) + uses: addnab/docker-run-action@v3 with: - command: make clang-tidy-quiet - max_attempts: 3 + image: px4io/px4-dev-clang:2021-09-08 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + git config --global --add safe.directory /workspace + make clang-tidy-quiet diff --git a/.github/workflows/compile_macos.yml b/.github/workflows/compile_macos.yml index 5c45a7710845..c9b83bdfec2b 100644 --- a/.github/workflows/compile_macos.yml +++ b/.github/workflows/compile_macos.yml @@ -19,13 +19,11 @@ jobs: ] steps: - name: install Python 3.10 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - - uses: actions/checkout@v1 - with: - token: ${{secrets.ACCESS_TOKEN}} + - uses: actions/checkout@v4 - name: setup run: ./Tools/setup/macos.sh; ./Tools/setup/macos.sh @@ -37,7 +35,7 @@ jobs: string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) message("::set-output name=timestamp::${current_date}") - name: ccache cache files - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.ccache key: macos_${{matrix.config}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} diff --git a/.github/workflows/ekf_functional_change_indicator.yml b/.github/workflows/ekf_functional_change_indicator.yml index a5fa9db59955..7fa9a834aa0d 100644 --- a/.github/workflows/ekf_functional_change_indicator.yml +++ b/.github/workflows/ekf_functional_change_indicator.yml @@ -1,21 +1,28 @@ name: EKF Change Indicator -on: pull_request +on: + pull_request: + branches: + - '*' jobs: unit_tests: runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 steps: - - uses: actions/checkout@v2.3.1 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: checkout newest version of branch - run: | - git fetch origin pull/${{github.event.pull_request.number}}/head:${{github.head_ref}} - git checkout ${GITHUB_HEAD_REF} + - name: main test - run: make tests TESTFILTER=EKF + uses: addnab/docker-run-action@v3 + with: + image: px4io/px4-dev-base-focal:2021-09-08 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + git config --global --add safe.directory /workspace + make tests TESTFILTER=EKF + - name: Check if there is a functional change run: git diff --exit-code working-directory: src/modules/ekf2/test/change_indication diff --git a/.github/workflows/ekf_update_change_indicator.yml b/.github/workflows/ekf_update_change_indicator.yml index 7de35207f096..682da5b42f4b 100644 --- a/.github/workflows/ekf_update_change_indicator.yml +++ b/.github/workflows/ekf_update_change_indicator.yml @@ -5,25 +5,41 @@ on: push jobs: unit_tests: runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 + # container: ghcr.io/px4/px4-dev:1.16.0-alpha2 env: GIT_COMMITTER_EMAIL: bot@px4.io GIT_COMMITTER_NAME: PX4BuildBot steps: - - uses: actions/checkout@v2.3.1 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: main test updates change indication files - run: make tests TESTFILTER=EKF + + - name: main test + uses: addnab/docker-run-action@v3 + with: + image: px4io/px4-dev-base-focal:2021-09-08 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + git config --global --add safe.directory /workspace + make tests TESTFILTER=EKF + - name: Check if there exists diff and save result in variable - run: echo "CHANGE_INDICATED=$(git diff --exit-code --output=/dev/null || echo $?)" >> $GITHUB_ENV + id: diff-check + run: echo "CHANGE_INDICATED=$(git diff --exit-code --output=/dev/null || echo $?)" >> $GITHUB_OUTPUT working-directory: src/modules/ekf2/test/change_indication + - name: auto-commit any changes to change indication uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: '[AUTO COMMIT] update change indication' + file_pattern: 'src/modules/ekf2/test/change_indication/*.csv' commit_user_name: ${GIT_COMMITTER_NAME} commit_user_email: ${GIT_COMMITTER_EMAIL} - - if: ${{env.CHANGE_INDICATED}} - name: if there is a functional change, fail check + commit_message: | + '[AUTO COMMIT] update change indication' + + See .github/workflopws/ekf_update_change_indicator.yml for more details + + - name: if there is a functional change, fail check + if: ${{ steps.diff-check.outputs.CHANGE_INDICATED }} run: exit 1 diff --git a/.github/workflows/failsafe_sim.yml b/.github/workflows/failsafe_sim.yml index 4d2ed21728a0..1868c8d79330 100644 --- a/.github/workflows/failsafe_sim.yml +++ b/.github/workflows/failsafe_sim.yml @@ -24,21 +24,27 @@ jobs: image: px4io/px4-dev-nuttx-focal:2022-08-12 options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} + - name: Install Node v20.18.0 + uses: actions/setup-node@v4 + with: + node-version: 20.18.0 - - name: check environment - run: | - export - ulimit -a - - name: install emscripten - run: | - git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk - cd _emscripten_sdk - ./emsdk install latest - ./emsdk activate latest - - name: ${{matrix.check}} - run: | - . ./_emscripten_sdk/emsdk_env.sh - make ${{matrix.check}} + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install empscripten + run: | + git clone https://github.com/emscripten-core/emsdk.git _emscripten_sdk + cd _emscripten_sdk + ./emsdk install latest + ./emsdk activate latest + + - name: Testing [${{ matrix.check }}] + # uses: addnab/docker-run-action@v3 + # with: + # image: px4io/px4-dev-base-focal:2021-09-08 + # options: -v ${{ github.workspace }}:/workspace + run: | + . ./_emscripten_sdk/emsdk_env.sh + make ${{ matrix.check }} diff --git a/.github/workflows/mavros_mission_tests.yml b/.github/workflows/mavros_mission_tests.yml index f1b6737334cb..b2807f988f6b 100644 --- a/.github/workflows/mavros_mission_tests.yml +++ b/.github/workflows/mavros_mission_tests.yml @@ -11,131 +11,26 @@ on: jobs: build: runs-on: ubuntu-latest - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true strategy: fail-fast: false matrix: config: - - {vehicle: "iris", mission: "MC_mission_box", build_type: "RelWithDebInfo"} - - {vehicle: "rover", mission: "rover_mission_1", build_type: "RelWithDebInfo"} - #- {vehicle: "plane", mission: "FW_mission_1", build_type: "RelWithDebInfo"} - #- {vehicle: "plane_catapult",mission: "FW_mission_1", build_type: "RelWithDebInfo"} - #- {vehicle: "standard_vtol", mission: "VTOL_mission_1", build_type: "Coverage"} - #- {vehicle: "standard_vtol", mission: "VTOL_mission_1", build_type: "AddressSanitizer"} - #- {vehicle: "tailsitter", mission: "VTOL_mission_1", build_type: "RelWithDebInfo"} - #- {vehicle: "tiltrotor", mission: "VTOL_mission_1", build_type: "RelWithDebInfo"} + - {vehicle: "iris", mission: "MC_mission_box"} + - {vehicle: "rover", mission: "rover_mission_1"} - container: - image: px4io/px4-dev-ros-melodic:2021-09-08 - options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 with: - token: ${{ secrets.ACCESS_TOKEN }} + fetch-depth: 0 - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") - - name: ccache cache files - uses: actions/cache@v2 + - name: Build SITL and Run Tests + uses: addnab/docker-run-action@v3 with: - path: ~/.ccache - key: sitl_tests-${{matrix.config.build_type}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} - restore-keys: sitl_tests-${{matrix.config.build_type}}-ccache- - - name: setup ccache - run: | - mkdir -p ~/.ccache - echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf - echo "compression = true" >> ~/.ccache/ccache.conf - echo "compression_level = 6" >> ~/.ccache/ccache.conf - echo "max_size = 100M" >> ~/.ccache/ccache.conf - echo "hash_dir = false" >> ~/.ccache/ccache.conf - ccache -s - ccache -z - - - name: check environment - run: | - export - ulimit -a - - name: Build PX4 and sitl_gazebo-classic - env: - PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}} - run: | - ccache -z - make px4_sitl_default - make px4_sitl_default sitl_gazebo-classic - ccache -s - - - name: Core dump settings - run: | - ulimit -c unlimited - echo "`pwd`/%e.core" > /proc/sys/kernel/core_pattern - - - name: Run SITL tests - env: - PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}} - run: | - export - ./test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=${{matrix.config.mission}} vehicle:=${{matrix.config.vehicle}} - timeout-minutes: 45 - - - name: Look at core files - if: failure() - run: gdb build/px4_sitl_default/bin/px4 px4.core -ex "thread apply all bt" -ex "quit" - - name: Upload px4 coredump - if: failure() - uses: actions/upload-artifact@v3 - with: - name: coredump - path: px4.core - - - name: ecl EKF analysis - if: always() - run: ./Tools/ecl_ekf/process_logdata_ekf.py ~/.ros/log/*/*.ulg || true - - - name: Upload logs to flight review - if: always() - run: ./Tools/upload_log.py -q --description "${GITHUB_WORKFLOW} ${GITHUB_RUN_ID}" --feedback "${GITHUB_WORKFLOW} ${GITHUB_RUN_ID} ${GITHUB_REPOSITORY} ${GITHUB_REF}" --source CI ~/.ros/log/*/*.ulg - - - name: Upload px4 binary - if: failure() - uses: actions/upload-artifact@v3 - with: - name: binary - path: build/px4_sitl_default/bin/px4 - - - name: Store PX4 log - if: failure() - uses: actions/upload-artifact@v3 - with: - name: px4_log - path: ~/.ros/log/*/*.ulg - - - name: Store ROS log - if: failure() - uses: actions/upload-artifact@v3 - with: - name: ros_log - path: ~/.ros/**/rostest-*.log - - # Report test coverage - - name: Upload coverage - if: contains(matrix.config.build_type, 'Coverage') - run: | - git config --global credential.helper "" # disable the keychain credential helper - git config --global --add credential.helper store # enable the local store credential helper - echo "https://x-access-token:${{ secrets.ACCESS_TOKEN }}@github.com" >> ~/.git-credentials # add credential - git config --global url."https://github.com/".insteadof git@github.com: # credentials add credential - mkdir -p coverage - lcov --directory build/px4_sitl_default --base-directory build/px4_sitl_default --gcov-tool gcov --capture -o coverage/lcov.info - - name: Upload coverage information to Codecov - if: contains(matrix.config.build_type, 'Coverage') - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: mavros_mission - file: coverage/lcov.info + image: px4io/px4-dev-ros-melodic:2021-09-08 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + git config --global --add safe.directory /workspace + make px4_sitl_default + make px4_sitl_default sitl_gazebo-classic + ./test/rostest_px4_run.sh mavros_posix_test_mission.test mission:=${{matrix.config.mission}} vehicle:=${{matrix.config.vehicle}} diff --git a/.github/workflows/mavros_offboard_tests.yml b/.github/workflows/mavros_offboard_tests.yml index c45c45ac0bc5..a968af24ef25 100644 --- a/.github/workflows/mavros_offboard_tests.yml +++ b/.github/workflows/mavros_offboard_tests.yml @@ -17,120 +17,21 @@ jobs: fail-fast: false matrix: config: - - {test_file: "mavros_posix_tests_offboard_posctl.test", vehicle: "iris", build_type: "RelWithDebInfo"} - #- {test_file: "mavros_posix_tests_offboard_attctl.test", vehicle: "iris", build_type: "RelWithDebInfo"} - #- {test_file: "mavros_posix_tests_offboard_rpyrt_ctl.test", vehicle: "iris", build_type: "RelWithDebInfo"} + - {test_file: "mavros_posix_tests_offboard_posctl.test", vehicle: "iris"} - container: - image: px4io/px4-dev-ros-melodic:2021-09-08 - options: --privileged --ulimit core=-1 --security-opt seccomp=unconfined steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 with: - token: ${{ secrets.ACCESS_TOKEN }} + fetch-depth: 0 - - name: Prepare ccache timestamp - id: ccache_cache_timestamp - shell: cmake -P {0} - run: | - string(TIMESTAMP current_date "%Y-%m-%d-%H;%M;%S" UTC) - message("::set-output name=timestamp::${current_date}") - - name: ccache cache files - uses: actions/cache@v2 + - name: Build PX4 and Run Tests + uses: addnab/docker-run-action@v3 with: - path: ~/.ccache - key: sitl_tests-${{matrix.config.build_type}}-ccache-${{steps.ccache_cache_timestamp.outputs.timestamp}} - restore-keys: sitl_tests-${{matrix.config.build_type}}-ccache- - - name: setup ccache - run: | - mkdir -p ~/.ccache - echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf - echo "compression = true" >> ~/.ccache/ccache.conf - echo "compression_level = 6" >> ~/.ccache/ccache.conf - echo "max_size = 100M" >> ~/.ccache/ccache.conf - echo "hash_dir = false" >> ~/.ccache/ccache.conf - ccache -s - ccache -z - - - name: check environment - run: | - export - ulimit -a - - name: Build PX4 and sitl_gazebo-classic - env: - PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}} - run: | - ccache -z - make px4_sitl_default - make px4_sitl_default sitl_gazebo-classic - ccache -s - - - name: Core dump settings - run: | - ulimit -c unlimited - echo "`pwd`/%e.core" > /proc/sys/kernel/core_pattern - - - name: Run SITL tests - env: - PX4_CMAKE_BUILD_TYPE: ${{matrix.config.build_type}} - run: | - export - ./test/rostest_px4_run.sh ${{matrix.config.test_file}} vehicle:=${{matrix.config.vehicle}} - timeout-minutes: 45 - - - name: Look at core files - if: failure() - run: gdb build/px4_sitl_default/bin/px4 px4.core -ex "thread apply all bt" -ex "quit" - - name: Upload px4 coredump - if: failure() - uses: actions/upload-artifact@v3 - with: - name: coredump - path: px4.core - - - name: ecl EKF analysis - if: always() - run: ./Tools/ecl_ekf/process_logdata_ekf.py ~/.ros/log/*/*.ulg || true - - - name: Upload logs to flight review - if: always() - run: ./Tools/upload_log.py -q --description "${GITHUB_WORKFLOW} ${GITHUB_RUN_ID}" --feedback "${GITHUB_WORKFLOW} ${GITHUB_RUN_ID} ${GITHUB_REPOSITORY} ${GITHUB_REF}" --source CI ~/.ros/log/*/*.ulg - - - name: Upload px4 binary - if: failure() - uses: actions/upload-artifact@v3 - with: - name: binary - path: build/px4_sitl_default/bin/px4 - - - name: Store PX4 log - if: failure() - uses: actions/upload-artifact@v3 - with: - name: px4_log - path: ~/.ros/log/*/*.ulg - - - name: Store ROS log - if: failure() - uses: actions/upload-artifact@v3 - with: - name: ros_log - path: ~/.ros/**/rostest-*.log - - # Report test coverage - - name: Upload coverage - if: contains(matrix.config.build_type, 'Coverage') - run: | - git config --global credential.helper "" # disable the keychain credential helper - git config --global --add credential.helper store # enable the local store credential helper - echo "https://x-access-token:${{ secrets.ACCESS_TOKEN }}@github.com" >> ~/.git-credentials # add credential - git config --global url."https://github.com/".insteadof git@github.com: # credentials add credential - mkdir -p coverage - lcov --directory build/px4_sitl_default --base-directory build/px4_sitl_default --gcov-tool gcov --capture -o coverage/lcov.info - - name: Upload coverage information to Codecov - if: contains(matrix.config.build_type, 'Coverage') - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - flags: mavros_offboard - file: coverage/lcov.info + image: px4io/px4-dev-ros-melodic:2021-09-08 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + git config --global --add safe.directory /workspace + make px4_sitl_default + make px4_sitl_default sitl_gazebo-classic + ./test/rostest_px4_run.sh ${{matrix.config.test_file}} vehicle:=${{matrix.config.vehicle}} diff --git a/.github/workflows/nuttx_env_config.yml b/.github/workflows/nuttx_env_config.yml index d151444ea9c1..5e8dd1752e2d 100644 --- a/.github/workflows/nuttx_env_config.yml +++ b/.github/workflows/nuttx_env_config.yml @@ -11,16 +11,19 @@ on: jobs: build: runs-on: ubuntu-latest - container: px4io/px4-dev-nuttx-focal:2022-08-12 + container: ghcr.io/px4/px4-dev:1.16.0-alpha2 strategy: matrix: config: [ px4_fmu-v5, ] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 with: - token: ${{secrets.ACCESS_TOKEN}} + fetch-depth: 0 + + - name: Git ownership workaround + run: git config --system --add safe.directory '*' - name: make ${{matrix.config}} env: diff --git a/.github/workflows/python_checks.yml b/.github/workflows/python_checks.yml index 9503fc99ebe2..3c273ba12497 100644 --- a/.github/workflows/python_checks.yml +++ b/.github/workflows/python_checks.yml @@ -12,14 +12,18 @@ jobs: build: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 with: - token: ${{ secrets.ACCESS_TOKEN }} + fetch-depth: 0 + - name: Install Python3 run: sudo apt-get install python3 python3-setuptools python3-pip -y + - name: Install tools run: python3 -m pip install mypy types-requests flake8 --break-system-packages + - name: Check MAVSDK test scripts with mypy run: $HOME/.local/bin/mypy --strict test/mavsdk_tests/*.py + - name: Check MAVSDK test scripts with flake8 run: $HOME/.local/bin/flake8 test/mavsdk_tests/*.py diff --git a/ROMFS/px4fmu_common/init.d/rcS b/ROMFS/px4fmu_common/init.d/rcS index 28804361d1ed..defd8bf2ccf6 100644 --- a/ROMFS/px4fmu_common/init.d/rcS +++ b/ROMFS/px4fmu_common/init.d/rcS @@ -224,7 +224,7 @@ else # Look for airframe in ROMFS . ${R}etc/init.d/rc.autostart - if [ ${VEHICLE_TYPE} == none ] + if [[ ${VEHICLE_TYPE} == none ]] then # Use external startup file if [ $STORAGE_AVAILABLE = yes ] @@ -235,7 +235,7 @@ else fi fi - if [ ${VEHICLE_TYPE} == none ] + if [[ ${VEHICLE_TYPE} == none ]] then echo "ERROR [init] No airframe file found for SYS_AUTOSTART value" param set SYS_AUTOSTART 0 diff --git a/Tools/setup/ubuntu.sh b/Tools/setup/ubuntu.sh index e2f215dc2d66..c5c551a7d9e0 100755 --- a/Tools/setup/ubuntu.sh +++ b/Tools/setup/ubuntu.sh @@ -83,6 +83,7 @@ sudo DEBIAN_FRONTEND=noninteractive apt-get -y --quiet --no-install-recommends i python3-pip \ python3-setuptools \ python3-wheel \ + python-is-python3 \ rsync \ shellcheck \ unzip \