From 12d42267423143b52cedeaf112314cf657f658d4 Mon Sep 17 00:00:00 2001 From: Jan Prochazka <90197375+P-R-O-C-H-Y@users.noreply.github.com> Date: Sun, 2 Jun 2024 23:32:38 +0200 Subject: [PATCH] ci(2.x): Save sizes to gh-pages --- .github/scripts/on-push.sh | 33 ++++++++++++--- .github/scripts/sketch_utils.sh | 75 ++++++++++++++++++++++++++++++++- .github/workflows/push.yml | 47 ++++++++++++++++++++- 3 files changed, 147 insertions(+), 8 deletions(-) diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh index 61e7e3243e6..64ff7851ba8 100755 --- a/.github/scripts/on-push.sh +++ b/.github/scripts/on-push.sh @@ -12,7 +12,8 @@ function build(){ local fqbn=$2 local chunk_index=$3 local chunks_cnt=$4 - shift; shift; shift; shift; + local build_log=$5 + shift; shift; shift; shift; shift; local sketches=$* local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build" @@ -25,6 +26,9 @@ function build(){ if [ "$OS_IS_LINUX" == "1" ]; then args+=" -p $ARDUINO_ESP32_PATH/libraries" args+=" -i $chunk_index -m $chunks_cnt" + if [ $build_log -eq 1 ]; then + args+=" -l $build_log" + fi ${BUILD_SKETCHES} ${args} else for sketch in ${sketches}; do @@ -48,6 +52,7 @@ fi CHUNK_INDEX=$1 CHUNKS_CNT=$2 +BUILD_LOG=$3 BUILD_PIO=0 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then CHUNK_INDEX=0 @@ -58,6 +63,10 @@ elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then BUILD_PIO=1 fi +if [ -z "$BUILD_LOG" ] || [ "$BUILD_LOG" -le 0 ]; then + BUILD_LOG=0 +fi + #echo "Updating submodules ..." #git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1 @@ -84,11 +93,25 @@ if [ "$BUILD_PIO" -eq 0 ]; then $ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino\ $ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino\ " + #create sizes_file + sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json" + + if [ "$BUILD_LOG" -eq 1 ]; then + #create sizes_file and echo start of JSON array with "boards" key + echo "{\"boards\": [" > $sizes_file + fi + + build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 + build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32XX + build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32XX + build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32 - build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 - build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX - build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX - build "esp32" $FQBN_ESP32 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32 + if [ "$BUILD_LOG" -eq 1 ]; then + #remove last comma from the last JSON object + sed -i '$ s/.$//' "$sizes_file" + #echo end of JSON array + echo "]}" >> $sizes_file + fi else source ${SCRIPTS_DIR}/install-platformio-esp32.sh # PlatformIO ESP32 Test diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index e490bec38df..b11112c4c84 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -27,6 +27,14 @@ function build_sketch(){ # build_sketch [ex shift sketchdir=$1 ;; + -i ) + shift + chunk_index=$1 + ;; + -l ) + shift + log_compilation=$1 + ;; * ) break ;; @@ -127,6 +135,9 @@ function build_sketch(){ # build_sketch [ex build_dir="$HOME/.arduino/tests/$sketchname/build.tmp" fi + output_file="$HOME/.arduino/cli_compile_output.txt" + sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json" + mkdir -p "$ARDUINO_CACHE_DIR" for i in `seq 0 $(($len - 1))` do @@ -151,7 +162,40 @@ function build_sketch(){ # build_sketch [ex --build-property "compiler.warning_flags.all=-Wall -Werror=all -Wextra" \ --build-cache-path "$ARDUINO_CACHE_DIR" \ --build-path "$build_dir" \ - $xtra_opts "${sketchdir}" + $xtra_opts "${sketchdir}" \ + > $output_file + + exit_status=$? + if [ $exit_status -ne 0 ]; then + echo ""ERROR: Compilation failed with error code $exit_status"" + exit $exit_status + fi + + if [ $log_compilation ]; then + #Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string + flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' $output_file | awk '{print $3}') + flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $5}' | tr -d '(%)') + ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' $output_file | awk '{print $4}') + ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $6}' | tr -d '(%)') + + # Extract the directory path excluding the filename + directory_path=$(dirname "$sketch") + # Define the constant part + constant_part="/home/runner/Arduino/hardware/espressif/esp32/libraries/" + # Extract the desired substring using sed + lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||") + #append json file where key is fqbn, sketch name, sizes -> extracted values + echo "{\"name\": \"$lib_sketch_name\", + \"sizes\": [{ + \"flash_bytes\": $flash_bytes, + \"flash_percentage\": $flash_percentage, + \"ram_bytes\": $ram_bytes, + \"ram_percentage\": $ram_percentage + }] + }," >> "$sizes_file" + fi + + elif [ -f "$ide_path/arduino-builder" ]; then echo "Building $sketchname with arduino-builder and FQBN=$currfqbn" echo "Build path = $build_dir" @@ -253,6 +297,10 @@ function build_sketches(){ # build_sketches > "$sizes_file" + fi + local sketchnum=0 - args+=" -ai $ide_path -au $user_path" + args+=" -ai $ide_path -au $user_path -i $chunk_index" + if [ $log_compilation ]; then + args+=" -l $log_compilation" + fi for sketch in $sketches; do local sketchdir=$(dirname $sketch) local sketchdirname=$(basename $sketchdir) @@ -334,6 +393,18 @@ function build_sketches(){ # build_sketches > "$sizes_file" + #echo end of board sizes_file json + echo "}," >> "$sizes_file" + fi + return 0 } diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index fecf743dfc9..ad4de3512e2 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -45,7 +45,15 @@ jobs: 'tools/get.py', '.github/scripts/install-arduino-ide.sh') }} - name: Build Sketches - run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15 + run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15 1 + + #Upload cli compile json as artifact + - name: Upload cli compile json + uses: actions/upload-artifact@v4 + with: + name: pr_cli_compile_${{ matrix.chunk }} + path: cli_compile_${{ matrix.chunk }}.json + overwrite: true # Windows and MacOS build-arduino-win-mac: @@ -106,3 +114,40 @@ jobs: idf.py create-project test echo CONFIG_FREERTOS_HZ=1000 > test/sdkconfig.defaults idf.py -C test -DEXTRA_COMPONENT_DIRS=$PWD/components build + + # Save 2.x artifacts to gh-pages + save-artifacts: + name: Save build artifacts + needs: build-arduino-linux + if: github.event_name == 'push' && github.ref == 'refs/heads/release/v2.x' + runs-on: ubuntu-latest + steps: + # Check out repository + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{secrets.GITHUB_TOKEN}} + fetch-depth: '0' + + - name: Switch branch + run: + git checkout remotes/origin/gh-pages + + - name: Download sketches reports artifact + uses: actions/download-artifact@v4 + with: + pattern: pr_cli_compile_* + merge-multiple: true + path: v2.x_cli_compile + + - name: List files in the directory + run: ls -R + + - name: Commit json files to gh-pages if on release branch + if: github.event_name == 'push' && github.ref == 'refs/heads/release/v2.x' + run: | + git config user.name github-actions + git config user.email github-actions@github.com + git add --all + git commit -m "Updated 2.x cli compile json files" + git push origin HEAD:gh-pages