diff --git a/.github/workflows/scrape-groceries.yml b/.github/workflows/scrape-groceries.yml index 389b9db..06952df 100644 --- a/.github/workflows/scrape-groceries.yml +++ b/.github/workflows/scrape-groceries.yml @@ -1,6 +1,6 @@ name: Scrape Groceries on: - push: + workflow_dispatch: schedule: - cron: "0 1 * * *" # Daily at 1am UTC (12pm AEST) env: @@ -20,13 +20,11 @@ jobs: name: woolies_snapshot path: ./output/woolies/ - name: Configure AWS Credentials - if: ${{ github.event_name == 'schedule' }} uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: "${{ vars.TARGET_ROLE_ARN }}" aws-region: ap-southeast-2 - run: aws s3 sync ./output/woolies/ s3://grocery-scrape-au/woolies/ - if: ${{ github.event_name == 'schedule' }} scrape-coles: permissions: contents: read # Required for checkout action @@ -41,10 +39,8 @@ jobs: name: coles_snapshot path: ./output/coles/ - name: Configure AWS Credentials - if: ${{ github.event_name == 'schedule' }} uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: "${{ vars.TARGET_ROLE_ARN }}" aws-region: ap-southeast-2 - - run: aws s3 sync ./output/coles/ s3://grocery-scrape-au/coles/ - if: ${{ github.event_name == 'schedule' }} \ No newline at end of file + - run: aws s3 sync ./output/coles/ s3://grocery-scrape-au/coles/ \ No newline at end of file diff --git a/.github/workflows/scrape-test.yml b/.github/workflows/scrape-test.yml new file mode 100644 index 0000000..ad43de2 --- /dev/null +++ b/.github/workflows/scrape-test.yml @@ -0,0 +1,24 @@ +name: Quick scrape test +on: + push: +jobs: + scrape-woolies: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: pip3 install -r requirements.txt + - run: python3 main.py --quick woolies + - uses: actions/upload-artifact@v3 + with: + name: woolies_snapshot + path: ./output/woolies/ + scrape-coles: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: pip3 install -r requirements.txt + - run: python3 main.py --quick coles + - uses: actions/upload-artifact@v3 + with: + name: coles_snapshot + path: ./output/coles/ \ No newline at end of file