Merge branch 'baidubce:master' into master

.env_template

@@ -0,0 +1,8 @@
+# rename this file to .env
+
+APPBUILDER_TOKEN=your-token
+APPBUILDER_TOKEN_V2=your-token
+BAIDU_VDB_API_KEY=your-token
+INSTANCE_ID=your-token
+DATASET_ID=your-token
+APPBUILDER_TOKEN_DOC_FORMAT=your-token

.github/workflows/python-package.yml

@@ -7,17 +7,16 @@ on:
   workflow_dispatch:
   push:
     branches: [ "master" ]
-  pull_request:
+  pull_request_target:
     branches: [ "master" ]
 
 jobs:
-  build:
-
+  Pylint:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9"]
+        python-version: ["3.12.5"]
 
     steps:
     - uses: actions/checkout@v3
@@ -29,10 +28,311 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         python -m pip install flake8 pytest
+        python -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+  Linux-CI-Tests:
+    runs-on: ubuntu-latest
+    env:
+      APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }}
+      APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }}
+      BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }}
+      INSTANCE_ID: ${{ secrets.INSTANCE_ID }}
+      DATASET_ID: ${{ secrets.DATASET_ID }}
+      APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9","3.12"]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Print ENV
+      run: |
+        env
+        ls
+        pwd
+    - name: Git Clone And Checkout Branch
+      run: |
+        fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git"
+        mkdir cicd
+        cd cicd
+        git clone $fork_repo
+        cd app-builder
+        git checkout $GITHUB_HEAD_REF
+        git remote add upstream https://github.com/baidubce/app-builder.git 
+        git fetch upstream
+        git remote -v
+        git status
+
+        # 找到当前分支与 upstream/master 的共同祖先提交
+        merge_base=$(git merge-base HEAD upstream/master)
+        echo "merge_base=$merge_base"
+
+        # 比较当前分支与 merge_base 之间的差异
+        changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base)
+        changed_files_py_sh=$(git diff --name-only --diff-filter=ACMRT $merge_base -- $(find appbuilder -type f \( -name '*.py' -o -name '*.sh' \)))
+        
+        echo "发生更改的文件为:"
+        echo "$changed_files"
+
+        echo "发生更改的py/sh文件为:"
+
+        if [ -n "$changed_files_py_sh" ]; then
+          export APPBUILDER_PYTHON_TESTS=True
+          echo "$changed_files_py_sh"
+        else
+          export APPBUILDER_PYTHON_TESTS=False
+          echo "没有检测到Python或Shell文件被更改"
+        fi
+
+        echo "APPBUILDER_PYTHON_TESTS=$APPBUILDER_PYTHON_TESTS" >> $GITHUB_ENV
+        pwd
+    - name: Install dependencies
+      run: |
+        echo $APPBUILDER_PYTHON_TESTS
+        if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then
+          echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Install dependencies部分。"
+        else
+          echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Install dependencies部分..."
+          cd cicd/app-builder
+          pwd
+          sudo apt-get install ffmpeg libavcodec-extra
+          python3 -m ensurepip --upgrade
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade setuptools
+          python3 -m pip install wheel
+          python3 -m pip install coverage
+          python3 -m pip install diff-cover
+          python3 -m pip install pydub
+          python3 -m pip install SQLAlchemy==2.0.31
+          python3 -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9
+          python3 -m pip install opentelemetry-exporter-otlp==1.23.0 opentelemetry-instrumentation==0.44b0 opentelemetry-sdk==1.23.0 opentelemetry-api==1.23.0
+          python3 -m pip install pandas==2.2.2
+        fi
+    - name: Build whl
+      run: |
+        echo $APPBUILDER_PYTHON_TESTS
+        if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then
+          echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Build whl部分。"
+        else
+          echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Build whl部分..."
+          cd cicd/app-builder     
+          pwd
+          python3 setup.py bdist_wheel
+          python3 -m pip uninstall -y appbuilder-sdk
+          python3 -m pip install dist/*.whl
+        fi
+    - name: Test with unittest
+      run: |
+        echo $APPBUILDER_PYTHON_TESTS
+        if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then
+          echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Test with unittest部分。"
+        else
+          echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Test with unittest部分..."
+          cd cicd/app-builder
+          pwd
+          sh appbuilder/tests/run_python_test.sh
+        fi
+  Go-Test-CI:
+    runs-on: ubuntu-latest
+    env:
+      APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }}
+      APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }}
+      BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }}
+      INSTANCE_ID: ${{ secrets.INSTANCE_ID }}
+      DATASET_ID: ${{ secrets.DATASET_ID }}
+      APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }}
+
+      APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }}
+      DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }}
+      DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }}
+    strategy:
+      fail-fast: false
+      matrix:
+        go-version: ['1.18']
+      max-parallel: 2
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Git Clone And Checkout Branch
+      run: |
+        fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git"
+        mkdir cicd
+        cd cicd
+        git clone $fork_repo
+        cd app-builder
+        echo "github_head的值为："
+        echo $GITHUB_HEAD_REF
+        git checkout $GITHUB_HEAD_REF
+        git remote add upstream https://github.com/baidubce/app-builder.git 
+        git fetch upstream
+        git remote -v
+        git status
+
+        # 找到当前分支与 upstream/master 的共同祖先提交
+        merge_base=$(git merge-base HEAD upstream/master)
+        echo "merge_base=$merge_base"
+
+        # 比较当前分支与 merge_base 之间的差异
+        changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base)
+        changed_files_go=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'go/**')
+
+        echo "发生更改的文件为:"
+        echo "$changed_files"
+        echo "发生更改的Go相关文件为:"
+
+        if [ -n "$changed_files_go" ]; then
+          export APPBUILDER_GO_TESTS=True
+          echo "$changed_files_go"
+        else
+          export APPBUILDER_GO_TESTS=False
+          echo "没有检测到Go或Shell文件被更改"
+        fi
+
+        echo "APPBUILDER_GO_TESTS=$APPBUILDER_GO_TESTS" >> $GITHUB_ENV
+    - name: Install dependencies and tools
+      run: |
+        echo $APPBUILDER_GO_TESTS
+        if [ "$APPBUILDER_GO_TESTS" = "False" ]; then
+          echo "环境变量APPBUILDER_GO_TESTS为False,没有检测到Go文件被更改，跳过Install dependencies and tools部分。"
+        else
+          echo "检测到Go文件被更改，准备启动Install dependencies and tools部分..."
+          cd cicd/app-builder/go/appbuilder
+          go mod tidy
+
+          go get github.com/axw/gocov/gocov  # 安装增量覆盖率工具
+
+          # 安装 golangci-lint
+          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.3
+          echo "$GOPATH/bin" >> $GITHUB_PATH
+
+        fi
+
+    - name: Run tests and calculate incremental coverage
+      run: |
+        echo $APPBUILDER_GO_TESTS
+        if [ "$APPBUILDER_GO_TESTS" = "False" ]; then
+          echo "环境变量APPBUILDER_GO_TESTS为False，没有检测到Go文件被更改，跳过Run tests部分。"
+        else
+          echo "检测到Go文件被更改，准备启动Run tests部分..."
+          cd cicd/app-builder/go
+          # 运行测试并生成覆盖率报告
+          chmod +x run_go_test.sh
+          ./run_go_test.sh
+        fi
+      shell: bash
+  Java-CI-Tests:
+    runs-on: ubuntu-latest
+    env:
+      APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }}
+      APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }}
+      BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }}
+      INSTANCE_ID: ${{ secrets.INSTANCE_ID }}
+      DATASET_ID: ${{ secrets.DATASET_ID }}
+      APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }}
+
+      APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }}
+      DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }}
+      DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }}
+    strategy:
+      fail-fast: false
+      matrix:
+        java-version: ['8']  # 使用 Java 11 和 17 作为示例
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Set up JDK ${{ matrix.java-version }}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{ matrix.java-version }}
+        distribution: 'temurin'
+
+    - name: Print ENV
+      run: |
+        env
+        ls
+        pwd
+
+    - name: Git Clone And Checkout Branch
+      run: |
+        fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git"
+        mkdir cicd
+        cd cicd
+        git clone $fork_repo
+        cd app-builder
+        git checkout $GITHUB_HEAD_REF
+        git remote add upstream https://github.com/baidubce/app-builder.git 
+        git fetch upstream
+        git remote -v
+        git status
+
+        # 找到当前分支与 upstream/master 的共同祖先提交
+        merge_base=$(git merge-base HEAD upstream/master)
+        echo "merge_base=$merge_base"
+
+        # 比较当前分支与 merge_base 之间的差异
+        changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base)
+        changed_files_java=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'java/**')
+        
+        echo "发生更改的文件为:"
+        echo "$changed_files"
+
+        echo "发生更改的Java相关文件为:"
+
+        if [ -n "$changed_files_java" ]; then
+          export APPBUILDER_JAVA_TESTS=True
+          echo "$changed_files_java"
+        else
+          export APPBUILDER_JAVA_TESTS=False
+          echo "没有检测到Java文件被更改"
+        fi
+
+        echo "APPBUILDER_JAVA_TESTS=$APPBUILDER_JAVA_TESTS" >> $GITHUB_ENV
+        pwd
+
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install gitpython lxml
+    - name: Install dependencies
+      run: |
+        echo $APPBUILDER_JAVA_TESTS
+        if [ "$APPBUILDER_JAVA_TESTS" = "False" ]; then
+          echo "环境变量APPBUILDER_JAVA_TESTS为False,没有检测到Java文件被更改,跳过Install dependencies部分。"
+        else
+          echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动单元测试Install dependencies部分..."
+          cd cicd/app-builder/java
+          mvn clean install -DskipTests
+        fi
+ 
+    - name: Run Java tests with Jacoco
+      if: env.APPBUILDER_JAVA_TESTS == 'True'
+      run: |
+        echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动Run Java tests with Jacoco部分..."
+        cd cicd/app-builder/java
+        bash print_coverage.sh
+

.gitignore

@@ -49,6 +49,9 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+# Unit test / production
+asr_test.pcm
+components_error_info.txt
 
 # Translations
 *.mo
@@ -124,3 +127,30 @@ FETCH_HEAD
 # vscode
 .vscode
 ./ppdiffusers/ppdiffusers/version.py
+
+
+# chainlit
+.chainlit
+# chainlit.md is generated by chainlit if not found
+chainlit.md 
+
+# sqlite
+*.db
+go/coverage.json
+.gitignore
+go/coveragess.xml
+go/coverage.out
+go/coverage_full.html
+java/coverage_diff.json
+.gitignore
+.gitignore
+go/analyze_coverage.py
+java/coverage_diff_custom.html
+.gitignore
+java/coverage_diff.html
+java/coverage_report.txt
+coverage_diff_custom.html
+go/coverage_diff.html
+.gitignore
+go/appbuilder/coverage_full.html
+go/appbuilder/coverage_diff.html