scrape.yml

name: scrape

on:
  schedule:
    - cron:  '11 * * * *' # Each hour At 11 min Ref https://crontab.guru/examples.html
  workflow_dispatch:

jobs:
  build:
    name: Build
    runs-on: ubuntu-latest
    steps:
      #----------------------------------------------
      #       check-out repo and set-up python     
      #----------------------------------------------
      - name: Check out repository
        uses: actions/checkout@v3
      - name: Set up python 
        uses: actions/setup-python@v3
        with:
          python-version: 3.8
      #----------------------------------------------
      #  -----  install & configure poetry  -----      
      #----------------------------------------------
      # Install Poetry 1.8.5
      - name: Install Poetry 1.8.5
        run: |
          curl -sSL https://install.python-poetry.org | python3 - --version 1.8.5
      #----------------------------------------------
      #       load cached venv if cache exists      
      #----------------------------------------------
#       - name: Load cached venv
#         id: cached-poetry-dependencies
#         uses: actions/cache@v2
#         with:
#           path: .venv
#           key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
      #----------------------------------------------
      # install dependencies if cache does not exist 
      #----------------------------------------------
      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root --no-dev
      #----------------------------------------------
      # run scrape
      #----------------------------------------------       
      - name: Scrape
        env: 
          TOKEN: ${{ secrets.TOKEN }}
        run: |
          poetry run python -c 'import os;print(os.environ)'
          poetry run python daily_scrape.py
          poetry run python monthly_scrape.py
          poetry run python fuel_types.py

      #----------------------------------------------
      # publish to github
      #----------------------------------------------

      - uses: mikeal/publish-to-github-action@master
        env:
          BRANCH_NAME: 'master'
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}