daily_job_board_scrape.yml

name: Daily Python Script

on:
  schedule:
    - cron: "0 7 * * *"
  workflow_dispatch:

jobs:
  run_script:
    name: Call Scrapy Script
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
          
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          
      - name: Run Scrapy script
        run: cd job_board_scraper && python run_job_scraper.py
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          HASHIDS_SALT: ${{ secrets.HASHIDS_SALT }}
          PAGES_TO_SCRAPE_QUERY: ${{ secrets.PAGES_TO_SCRAPE_QUERY }}
          PG_DATABASE: ${{ secrets.PG_DATABASE }}
          PG_HOST: ${{ secrets.PG_HOST }}
          PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
          PG_USER: ${{ secrets.PG_USER }}
          CHUNK_SIZE: ${{ secrets.CHUNK_SIZE }}
          RAW_HTML_S3_BUCKET: ${{ secrets.RAW_HTML_S3_BUCKET }}

  run_dbt:
    name: Run dbt
    needs: run_script
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
        
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt
    
      - name: Install dbt dependencies
        run: cd levergreen_dbt && dbt deps

      - name: Build dbt models
        run: cd levergreen_dbt && dbt run
        env:
          PG_DATABASE: ${{ secrets.PG_DATABASE }}
          PG_HOST: ${{ secrets.PG_HOST }}
          PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
          PG_USER: ${{ secrets.PG_USER }}