add build system ci

.github/workflows/release_maixcam.yml

@@ -30,11 +30,12 @@ jobs:
           whereis python3
           # export PATH=~/.local/bin/:$PATH
           # pull sipeed/MaixCDK repo here first
-          pwd_path=$(pwd)
+          maixpy_path=$(pwd)
+          maixpy_version=`git describe --tag`
           cd ~
           git clone https://github.com/sipeed/MaixCDK --depth=1
           export MAIXCDK_PATH=`pwd`/MaixCDK
-          cd $pwd_path
+          cd $maixpy_path
           python -m pip install -U pip setuptools wheel twine
           python -m pip install -r $MAIXCDK_PATH/requirements.txt
           python -m pip install pybind11-stubgen
@@ -64,6 +65,35 @@ jobs:
           release_path=dist/$release_name
           echo "release_path=$release_path" >> $GITHUB_OUTPUT
           echo "release_name=$release_name" >> $GITHUB_OUTPUT
+          echo "--------------------------------"
+          echo "-- Generate MaixCDK version file --"
+          echo "--------------------------------"
+          cd $MAIXCDK_PATH
+          maixcdk_rev=`git rev-parse HEAD`
+          maixcdk_version_name="maixcdk_version_${maixcdk_rev}.txt"
+          maixcdk_version_path="${MAIXCDK_PATH}/${maixcdk_version_name}"
+          echo "maixcdk_version_path=$maixcdk_version_path" >> $GITHUB_OUTPUT
+          echo "maixcdk_version_name=$maixcdk_version_name" >> $GITHUB_OUTPUT
+
+      - name: Build OS
+        id: build_os
+        run: |
+          echo "--------------------------------"
+          echo "--      Generate system       --"
+          echo "--------------------------------"
+          cd ${maixpy_path}/tools/os
+          chmod +x gen_os.sh
+          date_now=`date +"%Y-%m-%d"`
+          os_version_name="maixcam-${date_now}-maixpy-${maixpy_version}"
+          base_os_path=tmp/base_os.img.xz
+          python download_base_os.py -o ${base_os_path}
+          python download_builtin_files.py --unzip tmp/dl_builtin_files
+          builtin_files_dir=tmp/dl_builtin_files/sys_builtin_files
+          ./gen_os.sh $base_os_path $release_path $builtin_files_dir $os_version_name
+          os_filename=${os_version_name}.img.xz
+          os_filepath=`pwd`/tmp/$os_filename
+          echo "os_path=$os_filepath" >> $GITHUB_OUTPUT
+          echo "os_name=$os_filename" >> $GITHUB_OUTPUT
 
       - name: Build doc
         id: build_doc
@@ -100,6 +130,14 @@ jobs:
           tag: ${{ github.ref }}
           repo_token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Upload MaixCDK version txt to release assets
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file: ${{ steps.build_maixpy.outputs.maixcdk_version_path }}
+          asset_name: ${{ steps.build_maixpy.outputs.maixcdk_version_name }}
+          tag: ${{ github.ref }}
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Upload MaixPy MaixCAM to release assets
         uses: svenstaro/upload-release-action@v2
         with:
@@ -116,6 +154,14 @@ jobs:
           tag: ${{ github.ref }}
           repo_token: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Upload OS to release assets
+        uses: svenstaro/upload-release-action@v2
+        with:
+          file: ${{ steps.build_os.outputs.os_path }}
+          asset_name: ${{ steps.build_os.outputs.os_name }}
+          tag: ${{ github.ref }}
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Publish MaixPy to pypi.org
         run: |
           echo "[pypi]" > ~/.pypirc

.gitignore

@@ -175,3 +175,8 @@ docs/api/maix/
 docs/out/
 CMakeLists.txt
 !*/**/CMakeLists.txt
+tools/tmp/
+tools/os/tmp/
+tools/os/img_root/
+tools/os/img_boot/
+projects/apps/

projects/build_all.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+set -e
+
+function build_start()
+{
+    # test_script $1
+    # if [ $? -ne 0 ]; then
+    #     echo "Error: $1 failed to execute."
+    #     exit 1
+    # fi
+    cd $1
+    if [[ -f main.py ]]; then
+        rm -rf dist
+        maixtool release
+    else
+        maixcdk distclean
+        maixcdk release -p maixcam
+    fi
+    mkdir -p ../apps
+    cp -r dist/pack/* ../apps
+    cd ..
+}
+
+rm -rf apps/
+
+for dir in */; do
+  if [ -d "$dir" ]; then
+    if [[ "${dir}x" != "apps/x" ]]; then
+      echo "----- build ${dir} -----"
+      build_start "${dir%/}"
+      echo "----- build ${dir} done -----"
+    fi
+  fi
+done
+
+

tools/os/base_system_version.txt

@@ -0,0 +1 @@
+20240709

tools/os/download_base_os.py

@@ -0,0 +1,71 @@
+import os
+import sys
+import requests
+import subprocess
+
+curr_dir = os.path.abspath(os.path.dirname(__file__))
+
+def get_release_image_url(tag_name):
+    owner = "sipeed"
+    repo = "LicheeRV-Nano-Build"
+    url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+
+    # Make the request to the GitHub API
+    response = requests.get(url)
+
+    # Check if the request was successful
+    if response.status_code == 200:
+        releases = response.json()
+        # Print the release information
+        for release in releases:
+            if release['tag_name'] == tag_name:
+                for asset in release['assets']:
+                    if asset['name'].endswith(".img.xz"):
+                        return asset['browser_download_url'], asset['name']
+    else:
+        raise Exception(f"Failed to retrieve releases: {response.status_code}")
+    return None, None
+
+def download_file(url, output_path, force=False):
+    save_dir = os.path.dirname(output_path)
+    # Ensure the save directory exists
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    if os.path.exists(output_path) and not force:
+        print(f"-- file {os.path.basename(output_path)} already exists, skip download")
+        return
+    try:
+        # Use subprocess to call wget
+        subprocess.run(
+            ["wget", "--progress=dot:giga", "-O", output_path, url],
+            check=True
+        )
+        print(f"Download completed: {output_path}")
+    except subprocess.CalledProcessError as e:
+        print(f"An error occurred: {e}")
+
+def get_base_image(file_path = ""):
+    version_txt = os.path.join(curr_dir, "base_system_version.txt")
+    with open(version_txt, "r") as f:
+        version = f.readline().strip()
+    url, filename = get_release_image_url(version)
+    if not url:
+        raise Exception("get base system image download url failed")
+    print(f"-- download base system {filename} from url: {url}")
+    if not file_path:
+        file_path = os.path.join(curr_dir, "tmp", filename)
+    download_file(url, file_path)
+    return file_path
+
+
+if __name__ == "__main__":
+    import argparse
+    args_parser = argparse.ArgumentParser()
+    args_parser.add_argument(
+        "-o", "--out", help="download file path, default will save to tmp/***.img.xz", default=""
+    )
+    args = args_parser.parse_args()
+
+    base_system_path = get_base_image(args.out)
+    print(f"\n-- got base system: {base_system_path}")
+

tools/os/download_builtin_files.py

@@ -0,0 +1,97 @@
+import os
+import sys
+import requests
+import subprocess
+import tarfile
+import zipfile
+
+curr_dir = os.path.abspath(os.path.dirname(__file__))
+
+def get_release_builtin_files_url():
+    owner = "sipeed"
+    repo = "MaixPy"
+    url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+
+    # Make the request to the GitHub API
+    response = requests.get(url)
+
+    # Check if the request was successful
+    if response.status_code == 200:
+        releases = response.json()
+        releases = sorted(releases, key=lambda x: x['created_at'], reverse=True)
+        release = releases[0]
+        for asset in release['assets']:
+            if "builtin_files" in asset['name']:
+                if (not asset['name'].endswith(".zip")) and ((not asset['name'].endswith(".tar.xz"))) and (not asset['name'].endswith(".xz")):
+                    raise Exception(f"bultin_files only support .zip, .tar.xz, .xz format, but found {asset['name']}")
+                return asset['browser_download_url'], asset['name']
+    else:
+        raise Exception(f"Failed to retrieve releases: {response.status_code}")
+    return None, None
+
+def download_file(url, output_path, force=False):
+    save_dir = os.path.dirname(output_path)
+    # Ensure the save directory exists
+    if not os.path.exists(save_dir):
+        os.makedirs(save_dir)
+    if os.path.exists(output_path) and not force:
+        print(f"-- file {os.path.basename(output_path)} already exists, skip download")
+        return
+    try:
+        # Use subprocess to call wget
+        subprocess.run(
+            ["wget", "--progress=dot:giga", "-O", output_path, url],
+            check=True
+        )
+        print(f"Download completed: {output_path}")
+    except subprocess.CalledProcessError as e:
+        print(f"An error occurred: {e}")
+
+def unzip(file_path, unzip_path):
+    """
+    解压文件
+    支持 .xz .tar.xz .zip 格式
+    """
+    ext = os.path.splitext(file_path)[1]
+    if ext not in [".xz", ".tar.xz", ".zip"]:
+        raise Exception("file format not support unzip")
+    if not os.path.exists(unzip_path):
+        os.makedirs(unzip_path)
+
+    if file_path.endswith('.tar.xz') or file_path.endswith('.xz'):
+        with tarfile.open(file_path, 'r:xz') as tar:
+            tar.extractall(path=unzip_path, filter='data')
+    elif file_path.endswith('.zip'):
+        with zipfile.ZipFile(file_path, 'r') as zip_ref:
+            zip_ref.extractall(unzip_path)
+    else:
+        raise ValueError("Unsupported file format")
+
+def get_builtin_files(file_path = "", unzip_path=""):
+    url, filename = get_release_builtin_files_url()
+    if not url:
+        raise Exception("get base system image download url failed")
+    print(f"-- download os builtin files {filename} from url: {url}")
+    if not file_path:
+        file_path = os.path.join(curr_dir, "tmp", filename)
+    download_file(url, file_path)
+    if unzip_path:
+        unzip(file_path, unzip_path)
+        return unzip_path
+    return file_path
+
+
+if __name__ == "__main__":
+    import argparse
+    args_parser = argparse.ArgumentParser()
+    args_parser.add_argument(
+        "-o", "--out", help="download file path, default will save to tmp/***.img.xz", default="",
+    )
+    args_parser.add_argument(
+        "--unzip", help="directly unzip files to --out dir", type=str, default="",
+    )
+    args = args_parser.parse_args()
+
+    os_builtin_files = get_builtin_files(args.out, args.unzip)
+    print(f"\n-- got builtin files: {os_builtin_files}")
+

tools/os/gen_app_info.py

@@ -0,0 +1,70 @@
+'''
+    Usage: python gen_app_info.py apps_dir
+'''
+import os
+import yaml
+import sys
+
+app_info_content = '''
+[basic]
+version=1
+
+'''
+
+if len(sys.argv) > 1:
+    apps_dir = sys.argv[1]
+else:
+    apps_dir = os.path.abspath(os.path.dirname(__file__))
+
+app_info_path = os.path.join(apps_dir, "app.info")
+
+high_priority_apps = [
+    "app_store",
+    "settings"
+]
+
+apps_info = {}
+for name in os.listdir(apps_dir):
+    app_info_str = ""
+    app_dir = os.path.join(apps_dir, name)
+    if not os.path.isdir(app_dir):
+        continue
+    app_yaml_path = os.path.join(app_dir, "app.yaml")
+    with open(app_yaml_path, "r") as f:
+        app_info = yaml.safe_load(f)
+    if app_info["id"] == "launcher":
+        continue
+    app_info_str += f'[{app_info["id"]}]\n'
+    valid_keys = ["name", "version", "icon", "author", "desc"]
+    for k, v in app_info.items():
+        valid = False
+        for valid_k in valid_keys:
+            if k.startswith(valid_k):
+                valid = True
+                break
+        if not valid:
+            continue
+        app_info_str += f'{k}={v}\n'
+    if "main.py" in os.listdir(app_dir):
+        exec_path = "main.py"
+    else:
+        exec_path = app_info["id"]
+    app_info_str += f"exec={exec_path}\n"
+    app_info_str += "\n"
+    apps_info[app_info["id"]] = app_info_str
+
+for id in high_priority_apps:
+    if id in apps_info:
+        app_info_content += apps_info[id]
+
+li = []
+for id, content in apps_info.items():
+    if id in high_priority_apps:
+        continue
+    li.append((id, content))
+li = sorted(li, key=lambda x:x[0])
+for id, content in li:
+    app_info_content += content
+
+with open(app_info_path, "w", encoding="utf-8") as f:
+    f.write(app_info_content)