Deploy Batch Aggregation to Production (#789)

* Batch Aggregation (#785)

* Add Celery and a test route

* Add new dependencies

* Test task tests

* Docker updates

* Scripts folder

* Setup deploy to test env

* Link redis container via docker

* Modify test task

* Add redis service to test workflow

* Hook up services

* Fix test arguments

* flake8

* newline

* rename and refactor

* Taking a swing at extraction

* oops

* update .gitignore

* Remove deploy files

* Update .gitignore

* Clean up test tests

* Add router tests

* Extremely placeholder BA lib tests

* Only override local import

* First few batch agg specs

* Updates to BatchAggregation & tests

* less flake8y

* Add final POST message to Panoptes

* Flake

* flake

* Pull etag before atempting update

* Remove unnecessary mocks

* Assert result set, not method called

* clean up spec mocks

* Add permissions checking, fix some specs, refactor Panoptes update

* Flake

* Use os.path for platform independence

* Undeleting deploy template

* Batch aggregation staging deploy (#786)

* Add logging statements

* Update celery task namespace

* Add staging deployment template

* Clean up new resource names

* Build to a single docker image

* Rename deployment & use Panoptes staging in staging deploy

* Fix secret name

* Sringify ID in comparison to value returned from Panoptes

* Update test

* Fix mock data type

* Use client's admin mode

* Fix a couple filepaths

* Use UUID as tmpdir path

* Finish run if Panoptes is unupdateable

* When the update panoptes resource doesn't exist but the call is successful

* Use jsonify to set mimetype

* cast inputs to ints just in case

* Enable public access to new containers

* Deploy staging with action

* hound?

* test fixes

* new hound

* Use correct k8s secret

* Use tag deployment for production (#788)

* Use tag deployment for production

* Add batchagg resources to prod template

.github/workflows/deploy_app.yml → .github/workflows/deploy_production.yml

@@ -2,8 +2,8 @@ name: Deploy to Production
 
 on:
   push:
-    branches:
-      - master
+    tags:
+      - production-release
   workflow_dispatch:
 
 jobs:
@@ -21,7 +21,7 @@ jobs:
     uses: zooniverse/ci-cd/.github/workflows/deploy_app.yaml@main
     needs: build_and_push_image
     with:
-      app_name: aggregation-caesar
+      app_name: aggregation
       repo_name: aggregation-for-caesar
       commit_id: ${{ github.sha }}
       environment: production
@@ -38,7 +38,7 @@ jobs:
       commit_id: ${{ github.sha }}
       job_name: Deploy to Production / deploy_app
       status: ${{ needs.deploy_production.result }}
-      title: "Aggregation deploy complete"
+      title: "Aggregation Production deploy complete"
       title_link: "https://aggregation-caesar.zooniverse.org"
     secrets:
       slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/deploy_staging.yml

@@ -0,0 +1,44 @@
+name: Deploy to Staging
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  build_and_push_image:
+    name: Build and Push Image
+    uses: zooniverse/ci-cd/.github/workflows/build_and_push_image.yaml@main
+    with:
+      repo_name: aggregation-for-caesar
+      commit_id: ${{ github.sha }}
+      latest: true
+      build_args: "REVISION=${{ github.sha }}"
+
+  deploy_staging:
+    name: Deploy to Staging
+    uses: zooniverse/ci-cd/.github/workflows/deploy_app.yaml@main
+    needs: build_and_push_image
+    with:
+      app_name: aggregation
+      repo_name: aggregation-for-caesar
+      commit_id: ${{ github.sha }}
+      environment: staging
+      deploy_check: false
+    secrets:
+      creds: ${{ secrets.AZURE_AKS }}
+
+  slack_notification:
+    name: Slack notification
+    uses: zooniverse/ci-cd/.github/workflows/slack_notification.yaml@main
+    needs: deploy_staging
+    if: always()
+    with:
+      commit_id: ${{ github.sha }}
+      job_name: Deploy to Staging / deploy_app
+      status: ${{ needs.deploy_staging.result }}
+      title: "Aggregation Staging deploy complete"
+      title_link: "https://aggregation-staging.zooniverse.org"
+    secrets:
+      slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/python-versions.yml

@@ -9,6 +9,16 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
     strategy:
       matrix:
         python-version: [3.8, 3.9, "3.10", "3.11"]
@@ -30,11 +40,14 @@ jobs:
     - name: Run tests
       env:
         TRAVIS: true  # one test is skipped on CI and looks for this env value
+        REDIS_HOST: redis
+        CELERY_BROKER_URL: redis://localhost:6379/0
+        CELERY_RESULT_BACKEND: redis://localhost:6379/0
       run: |
         coverage run
         coverage report
     - name: Coveralls
       if: ${{ matrix.python-version == 3.10 }}
-      env: 
+      env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: coveralls --service=github

.gitignore

@@ -113,3 +113,5 @@ endpoints.yml
 
 .vscode/
 .noseids
+tmp/*
+.DS_Store

Dockerfile

@@ -21,10 +21,12 @@ COPY . .
 RUN pip install -U .[online,test,doc]
 
 # make documentation
-RUN /bin/bash -lc ./make_docs.sh
+RUN /bin/bash -lc ./scripts/make_docs.sh
+
+ADD ./ /usr/src/aggregation
 
 ARG REVISION=''
 ENV REVISION=$REVISION
 
 # load configs and start flask app
-CMD ["bash", "./start-flask.sh"]
+CMD ["bash", "./scripts/start-flask.sh"]

docker-compose.yml

@@ -1,19 +1,62 @@
-version: '2'
+version: '3'
 services:
   aggregation:
     build:
       context: ./
       args:
         REVISION: fake-git-sha-id
+    image: aggregation-for-caesar:local
     volumes:
-      - ./panoptes_aggregation:/usr/src/aggregation/panoptes_aggregation
+      - ./:/usr/src/aggregation
       - ~/.aws:/root/.aws
     environment:
       - AWS_REGION=${AWS_REGION}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
       - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}
       - AWS_SECURITY_TOKEN=${AWS_SECURITY_TOKEN}
-      - LISTEN_PORT=5000
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - FLASK_DEBUG=1
+      - FLASK_ENV=development
+      - LISTEN_PORT=4000
     ports:
-      - "5000:5000"
+      - "4000:4000"
+    links:
+      - redis:redis
+
+  worker:
+    image: aggregation-for-caesar:local
+    command: celery --app panoptes_aggregation.batch_aggregation.celery worker --loglevel=info
+    volumes:
+      - ./:/usr/src/aggregation
+    environment:
+      - FLASK_DEBUG=1
+      - APP_SETTINGS=project.server.config.DevelopmentConfig
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+      - FLASK_ENV=development
+    links:
+      - redis:redis
+    depends_on:
+      - redis
+
+  dashboard:
+    image: aggregation-for-caesar:local
+    command: celery --app panoptes_aggregation.batch_aggregation.celery flower --port=5555 --broker=redis://redis:6379/0
+    ports:
+      - 5556:5555
+    environment:
+      - FLASK_DEBUG=1
+      - APP_SETTINGS=project.server.config.DevelopmentConfig
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/0
+    links:
+      - redis:redis
+    depends_on:
+      - redis
+      - worker
+
+  redis:
+    image: redis
+    command: redis-server --appendonly yes