Daily Python Script #184

	name: Daily Python Script

	on:
	schedule:
	- cron: "0 7 * * *"
	workflow_dispatch:

	jobs:
	run_script:
	name: Call Scrapy Script
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt

	- name: Run Scrapy script
	run: cd job_board_scraper && python run_job_scraper.py
	env:
	AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
	AWS_REGION: ${{ secrets.AWS_REGION }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
	HASHIDS_SALT: ${{ secrets.HASHIDS_SALT }}
	PAGES_TO_SCRAPE_QUERY: ${{ secrets.PAGES_TO_SCRAPE_QUERY }}
	PG_DATABASE: ${{ secrets.PG_DATABASE }}
	PG_HOST: ${{ secrets.PG_HOST }}
	PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
	PG_USER: ${{ secrets.PG_USER }}
	RAW_HTML_S3_BUCKET: ${{ secrets.RAW_HTML_S3_BUCKET }}

	run_dbt:
	name: Run dbt
	needs: run_script
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v3
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.9'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt

	- name: Install dbt dependencies
	run: cd levergreen_dbt && dbt deps

	- name: Build dbt models
	run: cd levergreen_dbt && dbt run
	env:
	PG_DATABASE: ${{ secrets.PG_DATABASE }}
	PG_HOST: ${{ secrets.PG_HOST }}
	PG_PASSWORD: ${{ secrets.PG_PASSWORD }}
	PG_USER: ${{ secrets.PG_USER }}

Provide feedback