ci.yml

name: CI

on:
  push:
    branches:
      - develop-*
      - develop
      - master
  pull_request:
    branches:
      - develop-*
      - develop
      - master

env:
  SETTINGS_MODULE: crawler.config.test

jobs:
  black:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install pipenv
        run: |
          pip install pipenv
      - name: Install dependencies
        run: |
          pipenv sync --dev --system
      - name: Check format with black
        run: |
          # stop the build if there are black formatting errors
          python -m black --check .
  flake8:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install pipenv
        run: |
          pip install pipenv
      - name: Install dependencies
        run: |
          pipenv sync --dev --system
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8
  mypy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install pipenv
        run: |
          pip install pipenv
      - name: Install dependencies
        run: |
          pipenv sync --dev --system
      - name: Run mypy
        run: |
          python -m mypy .
  test:
    runs-on: ubuntu-latest
    services:
      mysql:
        image: mysql:8.0
        ports:
          - 3306:3306
        env:
          MYSQL_ALLOW_EMPTY_PASSWORD: yes
          MYSQL_DATABASE: unified_warehouse_test
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - uses: actions/cache@v1
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/Pipfile') }}
          restore-keys: |
            ${{ runner.os }}-pip-
      - name: Install pipenv
        run: |
          pip install pipenv
      - name: Install dependencies
        run: |
          pipenv sync --dev --system
      - name: Setup the test MLWH and Events databases
        run: |
          python setup_test_db.py
      - name: Start a testing SFTP server
        run: |
          docker run -p "3010:22" -v ${PWD}/test:/home/foo -d atmoz/sftp foo:pass:1001
      - name: Start MongoDB
        uses: supercharge/mongodb-github-action@1.7.0
        with:
          mongodb-version: 4.2
          mongodb-replica-set: heron_rs
      - name: Test with pytest
        run: |
          SFTP_SERVER=localhost SFTP_PORT=3010 python -m pytest -vx
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}