scrape #19800
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: scrape | |
| on: | |
| schedule: | |
| - cron: '11 * * * *' # Each hour At 11 min Ref https://crontab.guru/examples.html | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| steps: | |
| #---------------------------------------------- | |
| # check-out repo and set-up python | |
| #---------------------------------------------- | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| - name: Set up python | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: 3.8 | |
| #---------------------------------------------- | |
| # ----- install & configure poetry ----- | |
| #---------------------------------------------- | |
| # Install Poetry 1.8.5 | |
| - name: Install Poetry 1.8.5 | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - --version 1.8.5 | |
| #---------------------------------------------- | |
| # load cached venv if cache exists | |
| #---------------------------------------------- | |
| # - name: Load cached venv | |
| # id: cached-poetry-dependencies | |
| # uses: actions/cache@v2 | |
| # with: | |
| # path: .venv | |
| # key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} | |
| #---------------------------------------------- | |
| # install dependencies if cache does not exist | |
| #---------------------------------------------- | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root --no-dev | |
| #---------------------------------------------- | |
| # run scrape | |
| #---------------------------------------------- | |
| - name: Scrape | |
| env: | |
| TOKEN: ${{ secrets.TOKEN }} | |
| run: | | |
| poetry run python -c 'import os;print(os.environ)' | |
| poetry run python daily_scrape.py | |
| poetry run python monthly_scrape.py | |
| poetry run python fuel_types.py | |
| #---------------------------------------------- | |
| # publish to github | |
| #---------------------------------------------- | |
| - uses: mikeal/publish-to-github-action@master | |
| env: | |
| BRANCH_NAME: 'master' | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |