selfdrive_tests.yaml

name: selfdrive
on:
  push:
    branches-ignore:
      - 'testing-closet*'
  pull_request:

env:
  BASE_IMAGE: openpilot-base
  DOCKER_REGISTRY: ghcr.io/commaai

  DOCKER_LOGIN: docker login ghcr.io -u adeebshihadeh -p ${{ secrets.CONTAINER_TOKEN }}
  BUILD: |
      docker pull $(grep -iohP '(?<=^from)\s+\S+' Dockerfile.openpilot_base) || true
      docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
      docker build --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache $BASE_IMAGE /bin/sh -c
  UNIT_TEST: coverage run --append -m unittest discover

jobs:
  # TODO: once actions/cache supports read only mode, use the cache for all jobs
  build_release:
    name: build release
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    env:
      STRIPPED_DIR: tmppilot
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: release/check-submodules.sh
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Strip non-release files
      run: |
        mkdir $STRIPPED_DIR
        cp -pR --parents $(cat release/files_common) $STRIPPED_DIR
        cp -pR --parents $(cat release/files_tici) $STRIPPED_DIR
        cp -pR --parents $(cat release/files_pc) $STRIPPED_DIR
        cp Dockerfile.openpilot_base $STRIPPED_DIR

        # need this to build on x86
        cp -pR --parents third_party/libyuv third_party/snpe selfdrive/modeld/runners $STRIPPED_DIR
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Build openpilot and run checks
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py && \
                          python -m unittest discover selfdrive/car"
    - name: Cleanup scons cache
      run: |
          cd $STRIPPED_DIR
          ${{ env.RUN }} "scons -j$(nproc) && \
                          rm -rf /tmp/scons_cache/* && \
                          scons -j$(nproc) --cache-populate"

  #build_mac:
  #  name: build macos
  #  runs-on: macos-latest
  #  timeout-minutes: 60
  #  steps:
  #  - uses: actions/checkout@v2
  #    with:
  #      submodules: true
  #  - name: Determine pre-existing Homebrew packages
  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
  #    run: |
  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
  #      echo 'EOF' >> $GITHUB_ENV
  #  - name: Cache dependencies
  #    id: dependency-cache
  #    uses: actions/cache@v2
  #    with:
  #      path: |
  #        ~/.pyenv
  #        ~/.local/share/virtualenvs/
  #        /usr/local/Cellar
  #        ~/github_brew_cache_entries.txt
  #        /tmp/scons_cache
  #      key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'Pipfile*') }}
  #      restore-keys: macos-
  #  - name: Brew link restored dependencies
  #    run: |
  #      if [ -f ~/github_brew_cache_entries.txt ]; then
  #        while read pkg; do
  #          brew link --force "$pkg" # `--force` for keg-only packages
  #        done < ~/github_brew_cache_entries.txt
  #      else
  #        echo "Cache entries not found"
  #      fi
  #  - name: Install dependencies
  #    run: ./tools/mac_setup.sh
  #  - name: Build openpilot
  #    run: |
  #      source tools/openpilot_env.sh
  #      pipenv run selfdrive/manager/build.py
  #
  #      # cleanup scons cache
  #      rm -rf /tmp/scons_cache/
  #      pipenv run scons -j$(nproc) --cache-populate
  #  - name: Remove pre-existing Homebrew packages for caching
  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
  #    run: |
  #      cd /usr/local/Cellar
  #      new_cellar=$(ls -1)
  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
  #          rm -rf "$pkg"
  #        fi
  #      done
  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt

  build_webcam:
    name: build webcam
    runs-on: ubuntu-20.04
    timeout-minutes: 90
    env:
      IMAGE_NAME: openpilotwebcamci
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: |
        eval "$BUILD"
        docker pull $DOCKER_REGISTRY/$IMAGE_NAME:latest || true
        docker build --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/webcam/Dockerfile .
    - name: Build openpilot
      run: docker run --shm-size 1G --rm -v $PWD:/tmp/openpilot -e PYTHONPATH=/tmp/openpilot $DOCKER_REGISTRY/$IMAGE_NAME /bin/sh -c "cd /tmp/openpilot && USE_WEBCAM=1 scons -j$(nproc)"
    - name: Push to container registry
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest

  docker_push:
    name: docker push
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
    needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Push to container registry
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest

  static_analysis:
    name: static analysis
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build Docker image
      run: eval "$BUILD"
    - name: pre-commit
      run: ${{ env.RUN }} "git init && git add -A && pre-commit run --all"

  valgrind:
    name: valgrind
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/test_valgrind_replay.py') }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run valgrind
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
                        FILEREADER_CACHE=1 python selfdrive/test/test_valgrind_replay.py"
    - name: Print logs
      if: always()
      run: cat selfdrive/test/valgrind_logs.txt

  unit_tests:
    name: unit tests
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - name: Get current date
      id: date
      run: echo "::set-output name=time::$(date +'%s')"
    - name: Output timestamp
      run: echo $TIMESTAMP
      env:
        TIMESTAMP: ${{ steps.date.outputs.time }}
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: ${{ github.ref != 'refs/heads/master' || github.repository != 'commaai/openpilot' }}
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-${{ steps.date.outputs.time }}
        restore-keys: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run unit tests
      run: |
        ${{ env.RUN }} "scons -j$(nproc) --test && \
                        coverage run selfdrive/test/test_fingerprints.py && \
                        $UNIT_TEST common && \
                        $UNIT_TEST opendbc/can && \
                        $UNIT_TEST selfdrive/boardd && \
                        $UNIT_TEST selfdrive/controls && \
                        $UNIT_TEST selfdrive/monitoring && \
                        $UNIT_TEST selfdrive/loggerd && \
                        $UNIT_TEST selfdrive/car && \
                        $UNIT_TEST selfdrive/locationd && \
                        $UNIT_TEST selfdrive/athena && \
                        $UNIT_TEST selfdrive/thermald && \
                        $UNIT_TEST selfdrive/hardware/tici && \
                        $UNIT_TEST tools/lib/tests && \
                        ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
                        ./selfdrive/common/tests/test_util && \
                        ./selfdrive/common/tests/test_swaglog && \
                        ./selfdrive/loggerd/tests/test_logger &&\
                        ./selfdrive/proclogd/tests/test_proclog && \
                        ./selfdrive/ui/replay/tests/test_replay && \
                        ./selfdrive/camerad/test/ae_gray_test && \
                        coverage xml"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

  process_replay:
    name: process replay
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: ${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_processes.py') }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Run replay
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && \
                        FILEREADER_CACHE=1 CI=1 coverage run selfdrive/test/process_replay/test_processes.py && \
                        coverage xml"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2
    - name: Print diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt

  #model_replay:
  #  name: model replay
  #  runs-on: ubuntu-20.04
  #  timeout-minutes: 50
  #  steps:
  #  - uses: actions/checkout@v2
  #    with:
  #      submodules: true
  #  - name: Build Docker image
  #    run: eval "$BUILD"
  #  - name: Run replay
  #    run: |
  #      ${{ env.RUN }} "scons -j$(nproc) && \
  #                      selfdrive/test/process_replay/model_replay.py"

  test_longitudinal:
    name: longitudinal
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Test longitudinal
      run: |
        ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
                        scons -j$(nproc) && \
                        cd selfdrive/test/longitudinal_maneuvers && \
                        coverage run ./test_longitudinal.py && \
                        coverage xml"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2
    - uses: actions/upload-artifact@v2
      if: always()
      continue-on-error: true
      with:
        name: longitudinal
        path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/

  test_cars:
    name: cars
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    strategy:
      fail-fast: false
      matrix:
        job: [0, 1, 2, 3]
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Cache dependencies
      id: dependency-cache
      uses: actions/cache@v2
      with:
        path: /tmp/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/test/test_models.py', 'selfdrive/test/test_routes.py') }}-${{ matrix.job }}
    - name: Cache scons
      id: scons-cache
      # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
      env:
        CACHE_SKIP_SAVE: true
      with:
        path: /tmp/scons_cache
        key: scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
        restore-keys: |
          scons-${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}-
          scons-
    - name: Build Docker image
      run: eval "$BUILD"
    - name: Test car models
      run: |
        ${{ env.RUN }} "scons -j$(nproc) --test && \
                        FILEREADER_CACHE=1 coverage run -m pytest selfdrive/test/test_models.py && \
                        coverage xml && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
        NUM_JOBS: 4
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v2

  docs:
    name: build docs
    runs-on: ubuntu-20.04
    timeout-minutes: 50
    steps:
    - uses: actions/checkout@v2
      with:
        submodules: true
    - name: Build docker container
      run: |
        docker pull $DOCKER_REGISTRY/$BASE_IMAGE:latest || true
        docker pull $DOCKER_REGISTRY/openpilot-docs:latest || true
        DOCKER_BUILDKIT=1 docker build --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
    - name: Push docker container
      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
      run: |
        $DOCKER_LOGIN
        docker push $DOCKER_REGISTRY/openpilot-docs:latest