Adds example dockerfile.

lab-grid · Apr 15, 2021 · b21f3a7 · b21f3a7
1 parent 4101468
commit b21f3a7
Show file tree

Hide file tree

Showing 7 changed files with 312 additions and 22 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -10,13 +10,63 @@ jobs:
     name: Publish script-runner to pypi
     runs-on: ubuntu-latest
     steps:
-      - name: Publish a Python distribution to PyPI
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Prepare
+        id: prep
+        run: |
+          PYTHON_IMAGE_TAG=alpine
+          DOCKER_IMAGE=labflow/script-runner
+          VERSION=latest
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:${PYTHON_IMAGE_TAG}-latest,${DOCKER_IMAGE}:${PYTHON_IMAGE_TAG}-${VERSION}"
+          if [ "${{ github.event_name }}" = "push" ]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:${PYTHON_IMAGE_TAG}-sha-${GITHUB_SHA::8}"
+          fi
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}"
+          echo ::set-output name=pyversion::"${PYTHON_IMAGE_TAG}"
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel setuptools pipenv
+      #     pipenv lock --dev --requirements > requirements.dev.txt
+      #     pip install -r requirements.txt
+      # - name: Lint with flake8
+      #   run: |
+      #     # stop the build if there are Python syntax errors or undefined names
+      #     flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+      #     # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+      #     flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      # - name: Test with pytest
+      #   run: |
+      #     pytest
+      - name: Build python package
+        run: |
+          python setup.py sdist
+        env:
+          SCRIPT_RUNNER_VERSION: ${{ steps.prep.outputs.version }}
+      - name: Deploy to PyPI
+        if: success() && startsWith(github.ref, 'refs/tags')
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
 
   publish-docker:
+    needs: publish-pypi
+    if: startsWith(github.ref, 'refs/tags')
     name: Publish script-runner to dockerhub
     runs-on: ubuntu-latest
     steps:
@@ -64,7 +114,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: ./docker
-          file: ./Dockerfile
+          file: ./docker/Dockerfile
           builder: ${{ steps.buildx.outputs.name }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.prep.outputs.tags }}
@@ -78,10 +128,97 @@ jobs:
             SCRIPT_RUNNER_VERSION=${{ steps.prep.outputs.version }}
             PYTHON_IMAGE_TAG=${{ steps.prep.outputs.pyversion }}
 
+  publish-docker-example:
+    needs: publish-pypi
+    if: startsWith(github.ref, 'refs/tags')
+    name: Build and push script-runner-server-example docker image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=labflow/script-runner-example
+          VERSION=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}"
+          if [ "${{ github.event_name }}" = "push" ]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}"
+          fi
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}"
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/[email protected]
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Login to DockerHub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Push to Docker Hub
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          file: ./docker/Dockerfile.example
+          builder: ${{ steps.buildx.outputs.name }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.clone_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          build-args: |
+            SERVER_VERSION=${{ steps.prep.outputs.hashtag }}
+
   deploy-aws-example:
+    needs: publish-docker-example
+    if: startsWith(github.ref, 'refs/tags')
     name: Deploy swabseq-analysis-example to AWS
     runs-on: ubuntu-latest
     steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=labflow/script-runner-example
+          VERSION=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}"
+          if [ "${{ github.event_name }}" = "push" ]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}"
+          fi
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}"
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -92,28 +229,29 @@ jobs:
       - name: Setup Terraform tfvars
         uses: nowactions/envsubst@v1
         with:
-          input: ./examples/aws/example.tfvars
-          output: ./examples/aws/terraform.tfvars
+          input: ./terraform/aws/example.tfvars
+          output: ./terraform/aws/terraform.tfvars
         env:
           DNS_SUBDOMAIN: ${{secrets.DNS_SUBDOMAIN}}
           DNS_ZONE_ID: ${{secrets.DNS_ZONE_ID}}
+          IMAGE_TAG: ${{steps.plan.outputs.hashtag}}
 
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v1
 
       # - name: Terraform Format
       #   id: fmt
-      #   working-directory: ./examples/aws
+      #   working-directory: ./terraform/aws
       #   run: terraform fmt -check
 
       - name: Terraform Init
         id: init
-        working-directory: ./examples/aws
+        working-directory: ./terraform/aws
         run: terraform init
 
       - name: Terraform Plan
         id: plan
-        working-directory: ./examples/aws
+        working-directory: ./terraform/aws
         if: github.event_name == 'pull_request'
         run: terraform plan -no-color
         continue-on-error: true
@@ -150,14 +288,40 @@ jobs:
         run: exit 1
 
       - name: Terraform Apply
-        working-directory: ./examples/aws
+        working-directory: ./terraform/aws
         if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
         run: terraform apply -auto-approve
 
   deploy-azure-example:
+    needs: publish-docker-example
+    if: startsWith(github.ref, 'refs/tags')
     name: Deploy swabseq-analysis-example to Azure
     runs-on: ubuntu-latest
     steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=labflow/script-runner-example
+          VERSION=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}"
+          if [ "${{ github.event_name }}" = "push" ]; then
+            TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}"
+          fi
+          echo ::set-output name=version::${VERSION}
+          echo ::set-output name=tags::${TAGS}
+          echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+          echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}"
+
       - name: Configure Azure credentials
         uses: azure/login@v1
         with:
@@ -166,28 +330,29 @@ jobs:
       - name: Setup Terraform tfvars
         uses: nowactions/envsubst@v1
         with:
-          input: ./examples/azure/example.tfvars
-          output: ./examples/azure/terraform.tfvars
+          input: ./terraform/azure/example.tfvars
+          output: ./terraform/azure/terraform.tfvars
         env:
           DNS_SUBDOMAIN: ${{secrets.DNS_SUBDOMAIN}}
           DNS_ZONE_NAME: ${{secrets.DNS_ZONE_NAME}}
+          IMAGE_TAG: ${{steps.plan.outputs.hashtag}}
 
       - name: Setup Terraform
         uses: hashicorp/setup-terraform@v1
 
       # - name: Terraform Format
       #   id: fmt
-      #   working-directory: ./examples/azure
+      #   working-directory: ./terraform/azure
       #   run: terraform fmt -check
 
       - name: Terraform Init
         id: init
-        working-directory: ./examples/azure
+        working-directory: ./terraform/azure
         run: terraform init
 
       - name: Terraform Plan
         id: plan
-        working-directory: ./examples/azure
+        working-directory: ./terraform/azure
         if: github.event_name == 'pull_request'
         run: terraform plan -no-color
         continue-on-error: true
@@ -224,6 +389,6 @@ jobs:
         run: exit 1
 
       - name: Terraform Apply
-        working-directory: ./examples/azure
+        working-directory: ./terraform/azure
         if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main')
         run: terraform apply -auto-approve
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -0,0 +1,41 @@
+version: "3.7"
+services:
+  server:
+    image: labflow/script-runner-example:latest
+    build:
+      dockerfile: ./docker/Dockerfile.example
+      context: .
+      args:
+        SERVER_VERSION: local+devcontainer
+    command: "python3 -m flask run --host=0.0.0.0 --port=5000"
+    environment:
+      - FLASK_ENV=development
+      - PORT=5000
+      - PROPAGATE_EXCEPTIONS=True
+      - "SERVER_NAME=${SERVER_NAME}"
+      - "AUTH_PROVIDER=none"
+      - "CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379"
+      - "CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379"
+    ports:
+      - 5000:5000
+    volumes:
+      - ./:/app
+  worker:
+    image: labflow/script-runner-example:latest
+    build:
+      context: .
+      dockerfile: ./docker/Dockerfile.example
+      args:
+        SERVER_VERSION: local+devcontainer
+    command: "python3 -m celery -A script_runner.analysis worker"
+    environment:
+      - "CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379"
+      - "CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379"
+      - "COMMAND_RUNDIR_BASE=/base-rundir"
+    volumes:
+      - ./:/app
+  redis:
+    image: redis:6-alpine
+    command: redis-server --requirepass ${REDIS_PASSWORD}
+    ports:
+      - 6379:6379
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,18 +1,18 @@
-ARG SCRIPT_RUNNER_VERSION=local+container
 ARG PYTHON_IMAGE_TAG=alpine
 
-FROM python:${PYTHON_IMAGE_TAG}}
+FROM python:${PYTHON_IMAGE_TAG}
+
+ARG SCRIPT_RUNNER_VERSION=local+container
 
 WORKDIR /app
 
-RUN pip install script-runner-api==${SCRIPT_RUNNER_VERSION}
+RUN pip install \
+    script-runner-api==${SCRIPT_RUNNER_VERSION}
 
 COPY ./entrypoint.sh /entrypoint.sh
 RUN dos2unix /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 
-COPY ./ .
-
 ENV FLASK_APP=script_runner.main:app
 ENV SERVER_VERSION=$SERVER_VERSION