Skip to content

Daily Crawler

Daily Crawler #8

Workflow file for this run

name: Daily Crawler
on:
schedule:
- cron: '0 15 * * 0' # UTC 기준 매주 일요일 15:00 (한국 시간 월요일 자정)
workflow_dispatch: # 수동 실행 옵션
jobs:
crawl:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: '3.12.4'
- name: Cache pip packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.txt
- name: Install Chrome
uses: browser-actions/setup-chrome@latest
- name: Check directory before crawling
run: ls -R
- name: Run crawler
run: python crawler.py --verbose
env:
PYTHONUNBUFFERED: 1
- name: Check directory after crawling
run: ls -R
- name: Upload crawling results
uses: actions/upload-artifact@v3
if: always()
with:
name: crawling-results
path: dataset/
- name: Commit and push if changed
run: |
git config --global user.email "[email protected]"
git config --global user.name "xeros"
git add -A
git diff --quiet && git diff --staged --quiet || (git commit -m "Update data" && git push)