ci(2.x): Save sizes to gh-pages

espressif · Jun 2, 2024 · 12d4226 · 12d4226
1 parent dc596f1
commit 12d4226
Show file tree

Hide file tree

Showing 3 changed files with 147 additions and 8 deletions.
diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh
@@ -12,7 +12,8 @@ function build(){
     local fqbn=$2
     local chunk_index=$3
     local chunks_cnt=$4
-    shift; shift; shift; shift;
+    local build_log=$5
+    shift; shift; shift; shift; shift;
     local sketches=$*
 
     local BUILD_SKETCH="${SCRIPTS_DIR}/sketch_utils.sh build"
@@ -25,6 +26,9 @@ function build(){
     if [ "$OS_IS_LINUX" == "1" ]; then
         args+=" -p $ARDUINO_ESP32_PATH/libraries"
         args+=" -i $chunk_index -m $chunks_cnt"
+        if [ $build_log -eq 1 ]; then
+            args+=" -l $build_log"
+        fi
         ${BUILD_SKETCHES} ${args}
     else
         for sketch in ${sketches}; do
@@ -48,6 +52,7 @@ fi
 
 CHUNK_INDEX=$1
 CHUNKS_CNT=$2
+BUILD_LOG=$3
 BUILD_PIO=0
 if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then
     CHUNK_INDEX=0
@@ -58,6 +63,10 @@ elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then
     BUILD_PIO=1
 fi
 
+if [ -z "$BUILD_LOG" ] || [ "$BUILD_LOG" -le 0 ]; then
+    BUILD_LOG=0
+fi
+
 #echo "Updating submodules ..."
 #git -C "$GITHUB_WORKSPACE" submodule update --init --recursive > /dev/null 2>&1
 
@@ -84,11 +93,25 @@ if [ "$BUILD_PIO" -eq 0 ]; then
       $ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino\
       $ARDUINO_ESP32_PATH/libraries/Insights/examples/MinimalDiagnostics/MinimalDiagnostics.ino\
     "
+    #create sizes_file
+    sizes_file="$GITHUB_WORKSPACE/cli_compile_$CHUNK_INDEX.json"
+
+    if [ "$BUILD_LOG" -eq 1 ]; then
+        #create sizes_file and echo start of JSON array with "boards" key
+        echo "{\"boards\": [" > $sizes_file
+    fi
+
+    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
+    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32XX
+    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32XX
+    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $BUILD_LOG $SKETCHES_ESP32
 
-    build "esp32s3" $FQBN_ESP32S3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
-    build "esp32s2" $FQBN_ESP32S2 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
-    build "esp32c3" $FQBN_ESP32C3 $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32XX
-    build "esp32"   $FQBN_ESP32   $CHUNK_INDEX $CHUNKS_CNT $SKETCHES_ESP32
+    if [ "$BUILD_LOG" -eq 1 ]; then
+        #remove last comma from the last JSON object
+        sed -i '$ s/.$//' "$sizes_file"
+        #echo end of JSON array
+        echo "]}" >> $sizes_file
+    fi
 else
     source ${SCRIPTS_DIR}/install-platformio-esp32.sh
     # PlatformIO ESP32 Test

diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh
@@ -27,6 +27,14 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
             shift
             sketchdir=$1
             ;;
+        -i )
+            shift
+            chunk_index=$1
+            ;;
+        -l )
+            shift
+            log_compilation=$1
+            ;;
         * )
             break
             ;;
@@ -127,6 +135,9 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
         build_dir="$HOME/.arduino/tests/$sketchname/build.tmp"
     fi
 
+    output_file="$HOME/.arduino/cli_compile_output.txt"
+    sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
+
     mkdir -p "$ARDUINO_CACHE_DIR"
     for i in `seq 0 $(($len - 1))`
     do
@@ -151,7 +162,40 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
                 --build-property "compiler.warning_flags.all=-Wall -Werror=all -Wextra" \
                 --build-cache-path "$ARDUINO_CACHE_DIR" \
                 --build-path "$build_dir" \
-                $xtra_opts "${sketchdir}"
+                $xtra_opts "${sketchdir}" \
+                > $output_file
+
+            exit_status=$?
+            if [ $exit_status -ne 0 ]; then
+                echo ""ERROR: Compilation failed with error code $exit_status""
+                exit $exit_status
+            fi
+
+            if [ $log_compilation ]; then
+                #Extract the program storage space and dynamic memory usage in bytes and percentage in separate variables from the output, just the value without the string
+                flash_bytes=$(grep -oE 'Sketch uses ([0-9]+) bytes' $output_file | awk '{print $3}')
+                flash_percentage=$(grep -oE 'Sketch uses ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $5}' | tr -d '(%)')
+                ram_bytes=$(grep -oE 'Global variables use ([0-9]+) bytes' $output_file | awk '{print $4}')
+                ram_percentage=$(grep -oE 'Global variables use ([0-9]+) bytes \(([0-9]+)%\)' $output_file | awk '{print $6}' | tr -d '(%)')
+
+                # Extract the directory path excluding the filename
+                directory_path=$(dirname "$sketch")
+                # Define the constant part
+                constant_part="/home/runner/Arduino/hardware/espressif/esp32/libraries/"
+                # Extract the desired substring using sed
+                lib_sketch_name=$(echo "$directory_path" | sed "s|$constant_part||")
+                #append json file where key is fqbn, sketch name, sizes -> extracted values
+                echo "{\"name\": \"$lib_sketch_name\",
+                    \"sizes\": [{
+                            \"flash_bytes\": $flash_bytes,
+                            \"flash_percentage\": $flash_percentage,
+                            \"ram_bytes\": $ram_bytes,
+                            \"ram_percentage\": $ram_percentage
+                            }]
+                    }," >> "$sizes_file"
+            fi    
+
+
         elif [ -f "$ide_path/arduino-builder" ]; then
             echo "Building $sketchname with arduino-builder and FQBN=$currfqbn"
             echo "Build path = $build_dir"
@@ -253,6 +297,10 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             shift
             chunk_max=$1
             ;;
+        -l )
+            shift
+            log_compilation=$1
+            ;;
         * )
             break
             ;;
@@ -316,8 +364,19 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     echo "Start Sketch: $start_num"
     echo "End Sketch  : $end_index"
 
+    sizes_file="$GITHUB_WORKSPACE/cli_compile_$chunk_index.json"
+    if [ $log_compilation ]; then
+        #echo board,target and start of sketches to sizes_file json
+        echo "{ \"board\": \"$fqbn\",
+                \"target\": \"$target\",
+                \"sketches\": [" >> "$sizes_file"
+    fi
+
     local sketchnum=0
-    args+=" -ai $ide_path -au $user_path"
+    args+=" -ai $ide_path -au $user_path -i $chunk_index"
+    if [ $log_compilation ]; then
+        args+=" -l $log_compilation"
+    fi
     for sketch in $sketches; do
         local sketchdir=$(dirname $sketch)
         local sketchdirname=$(basename $sketchdir)
@@ -334,6 +393,18 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             return $result
         fi
     done
+
+    if [ $log_compilation ]; then
+        #remove last comma from json
+        if [ $i -eq $(($len - 1)) ]; then
+            sed -i '$ s/.$//' "$sizes_file"
+        fi
+        #echo end of sketches sizes_file json
+        echo "]" >> "$sizes_file"
+        #echo end of board sizes_file json
+        echo "}," >> "$sizes_file"
+    fi
+
     return 0
 }
 

diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
@@ -45,7 +45,15 @@ jobs:
                                             'tools/get.py',
                                             '.github/scripts/install-arduino-ide.sh') }}
     - name: Build Sketches
-      run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15
+      run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} 15 1
+
+    #Upload cli compile json as artifact
+    - name: Upload cli compile json
+      uses: actions/upload-artifact@v4
+      with:
+        name: pr_cli_compile_${{ matrix.chunk }}
+        path: cli_compile_${{ matrix.chunk }}.json
+        overwrite: true
 
   # Windows and MacOS
   build-arduino-win-mac:
@@ -106,3 +114,40 @@ jobs:
           idf.py create-project test
           echo CONFIG_FREERTOS_HZ=1000 > test/sdkconfig.defaults
           idf.py -C test -DEXTRA_COMPONENT_DIRS=$PWD/components build
+
+  # Save 2.x artifacts to gh-pages
+  save-artifacts:
+    name: Save build artifacts
+    needs: build-arduino-linux
+    if: github.event_name == 'push' && github.ref == 'refs/heads/release/v2.x'
+    runs-on: ubuntu-latest
+    steps:
+    # Check out repository
+      - name: Checkout repository    
+        uses: actions/checkout@v4
+        with:
+          token: ${{secrets.GITHUB_TOKEN}}
+          fetch-depth: '0'
+
+      - name: Switch branch
+        run:
+          git checkout remotes/origin/gh-pages
+
+      - name: Download sketches reports artifact
+        uses: actions/download-artifact@v4
+        with:
+          pattern: pr_cli_compile_*
+          merge-multiple: true
+          path: v2.x_cli_compile
+
+      - name: List files in the directory
+        run: ls -R
+
+      - name: Commit json files to gh-pages if on release branch
+        if: github.event_name == 'push' && github.ref == 'refs/heads/release/v2.x'
+        run: |
+          git config user.name github-actions
+          git config user.email [email protected]
+          git add --all
+          git commit -m "Updated 2.x cli compile json files"
+          git push origin HEAD:gh-pages