From 98db6e0b4819a844ed1aca64cc96624d101c5714 Mon Sep 17 00:00:00 2001 From: Robert Chu Date: Wed, 14 Apr 2021 19:23:02 -0700 Subject: [PATCH] Adds example dockerfile. --- .github/workflows/publish.yml | 116 +++++++++++++++++++++++++++++----- docker-compose.yaml | 41 ++++++++++++ docker/Dockerfile | 8 +-- docker/Dockerfile.example | 84 ++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 docker/Dockerfile.example diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3134387..3f03492 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,13 +10,37 @@ jobs: name: Publish script-runner to pypi runs-on: ubuntu-latest steps: - - name: Publish a Python distribution to PyPI + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools pipenv + # pipenv lock --dev --requirements > requirements.dev.txt + # pip install -r requirements.txt + # - name: Lint with flake8 + # run: | + # # stop the build if there are Python syntax errors or undefined names + # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # - name: Test with pytest + # run: | + # pytest + - name: Build python package + run: | + python setup.py sdist + - name: Deploy to PyPI + if: success() && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} publish-docker: + if: startsWith(github.ref, 'refs/tags') name: Publish script-runner to dockerhub runs-on: ubuntu-latest steps: @@ -63,8 +87,8 @@ jobs: - name: Push to Docker Hub uses: docker/build-push-action@v2 with: - context: ./docker - file: ./Dockerfile + context: ./ + file: ./docker/Dockerfile builder: ${{ steps.buildx.outputs.name }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.prep.outputs.tags }} @@ -78,7 +102,68 @@ jobs: SCRIPT_RUNNER_VERSION=${{ steps.prep.outputs.version }} PYTHON_IMAGE_TAG=${{ steps.prep.outputs.pyversion }} + publish-docker-example: + if: startsWith(github.ref, 'refs/tags') + name: Build and push script-runner-server-example docker image + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=labflow/script-runner-example + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}" + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=version::${VERSION} + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}" + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2.1.4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Push to Docker Hub + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./docker/Dockerfile.example + builder: ${{ steps.buildx.outputs.name }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + SERVER_VERSION=${{ steps.prep.outputs.hashtag }} + deploy-aws-example: + if: startsWith(github.ref, 'refs/tags') name: Deploy swabseq-analysis-example to AWS runs-on: ubuntu-latest steps: @@ -92,8 +177,8 @@ jobs: - name: Setup Terraform tfvars uses: nowactions/envsubst@v1 with: - input: ./examples/aws/example.tfvars - output: ./examples/aws/terraform.tfvars + input: ./terraform/aws/example.tfvars + output: ./terraform/aws/terraform.tfvars env: DNS_SUBDOMAIN: ${{secrets.DNS_SUBDOMAIN}} DNS_ZONE_ID: ${{secrets.DNS_ZONE_ID}} @@ -103,17 +188,17 @@ jobs: # - name: Terraform Format # id: fmt - # working-directory: ./examples/aws + # working-directory: ./terraform/aws # run: terraform fmt -check - name: Terraform Init id: init - working-directory: ./examples/aws + working-directory: ./terraform/aws run: terraform init - name: Terraform Plan id: plan - working-directory: ./examples/aws + working-directory: ./terraform/aws if: github.event_name == 'pull_request' run: terraform plan -no-color continue-on-error: true @@ -150,11 +235,12 @@ jobs: run: exit 1 - name: Terraform Apply - working-directory: ./examples/aws + working-directory: ./terraform/aws if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') run: terraform apply -auto-approve deploy-azure-example: + if: startsWith(github.ref, 'refs/tags') name: Deploy swabseq-analysis-example to Azure runs-on: ubuntu-latest steps: @@ -166,8 +252,8 @@ jobs: - name: Setup Terraform tfvars uses: nowactions/envsubst@v1 with: - input: ./examples/azure/example.tfvars - output: ./examples/azure/terraform.tfvars + input: ./terraform/azure/example.tfvars + output: ./terraform/azure/terraform.tfvars env: DNS_SUBDOMAIN: ${{secrets.DNS_SUBDOMAIN}} DNS_ZONE_NAME: ${{secrets.DNS_ZONE_NAME}} @@ -177,17 +263,17 @@ jobs: # - name: Terraform Format # id: fmt - # working-directory: ./examples/azure + # working-directory: ./terraform/azure # run: terraform fmt -check - name: Terraform Init id: init - working-directory: ./examples/azure + working-directory: ./terraform/azure run: terraform init - name: Terraform Plan id: plan - working-directory: ./examples/azure + working-directory: ./terraform/azure if: github.event_name == 'pull_request' run: terraform plan -no-color continue-on-error: true @@ -224,6 +310,6 @@ jobs: run: exit 1 - name: Terraform Apply - working-directory: ./examples/azure + working-directory: ./terraform/azure if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') run: terraform apply -auto-approve diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..10af756 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,41 @@ +version: "3.7" +services: + server: + image: labflow/script-runner-example:latest + build: + dockerfile: ./docker/Dockerfile.example + context: . + args: + SERVER_VERSION: local+devcontainer + command: "python3 -m flask run --host=0.0.0.0 --port=5000" + environment: + - FLASK_ENV=development + - PORT=5000 + - PROPAGATE_EXCEPTIONS=True + - "SERVER_NAME=${SERVER_NAME}" + - "AUTH_PROVIDER=none" + - "CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379" + - "CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379" + ports: + - 5000:5000 + volumes: + - ./:/app + worker: + image: labflow/script-runner-example:latest + build: + context: . + dockerfile: ./docker/Dockerfile.example + args: + SERVER_VERSION: local+devcontainer + command: "python3 -m celery -A script_runner.analysis worker" + environment: + - "CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379" + - "CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379" + - "COMMAND_RUNDIR_BASE=/base-rundir" + volumes: + - ./:/app + redis: + image: redis:6-alpine + command: redis-server --requirepass ${REDIS_PASSWORD} + ports: + - 6379:6379 diff --git a/docker/Dockerfile b/docker/Dockerfile index bd28b6c..65b1245 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,18 +1,18 @@ -ARG SCRIPT_RUNNER_VERSION=local+container ARG PYTHON_IMAGE_TAG=alpine FROM python:${PYTHON_IMAGE_TAG}} +ARG SCRIPT_RUNNER_VERSION=local+container + WORKDIR /app -RUN pip install script-runner-api==${SCRIPT_RUNNER_VERSION} +RUN pip install \ + script-runner-api==${SCRIPT_RUNNER_VERSION} COPY ./entrypoint.sh /entrypoint.sh RUN dos2unix /entrypoint.sh RUN chmod +x /entrypoint.sh -COPY ./ . - ENV FLASK_APP=script_runner.main:app ENV SERVER_VERSION=$SERVER_VERSION diff --git a/docker/Dockerfile.example b/docker/Dockerfile.example new file mode 100644 index 0000000..ad3247a --- /dev/null +++ b/docker/Dockerfile.example @@ -0,0 +1,84 @@ +ARG SERVER_VERSION=local+container +ARG SCRIPT_RUNNER_VERSION=local+container + +# Start with rocker/tidyverse base image +FROM rocker/verse:3.6.3 + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# get and install bcl2fastq +RUN cd /tmp \ + && wget https://www.dropbox.com/s/idi0xfu0thurk7q/bcl2fastq2-v2-20-0-linux-x86-64.zip \ + && unzip bcl2fastq2-v2-20-0-linux-x86-64.zip -p \ + | tar xvf bcl2fastq2-v2.20.0.422-Linux-x86_64.rpm -C / usr/local/bin/bcl2fastq \ + && rm /tmp/bcl2fastq2* + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Install extra *nix utils +# x11, mesa, glu1 are so we can install paletteer +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + libpq-dev \ + pigz \ + vim \ + git \ + less \ + curl \ + wget \ + parallel \ + python3-pip \ + bzip2 \ + libcairo2-dev \ + libfontconfig1-dev \ + ca-certificates \ + dos2unix \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# R Deps +RUN install2.r --error \ + BiocManager \ + seqinr \ + viridis \ + GGally \ + reader \ + plater \ + XML \ + DT \ + glmnet \ + speedglm \ + sandwich \ + ggbeeswarm \ + stringdist \ + argparser \ + kableExtra + +# Install bioconductor packages +RUN R --slave -e "BiocManager::install(c('savR', 'edgeR', 'qvalue', 'ShortRead', 'Rqc'))" + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# swabseq-analysis script +RUN git clone git://github.com/lab-grid/swabseq-analysis /app + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Python Env +WORKDIR /app + +RUN pip3 install \ + pandas \ + script-runner-api==${SCRIPT_RUNNER_VERSION} + +COPY ./entrypoint.sh /entrypoint.sh +RUN dos2unix /entrypoint.sh +RUN chmod +x /entrypoint.sh + +RUN mkdir /base-rundir +RUN wget -qO- https://swabseq-analysis-examples.s3-us-west-1.amazonaws.com/bcls/H3FY3K.tar.gz \ + | tar xvz -C /base-rundir + +ENV PYTHONPATH="${RBASE}:${PYTHONPATH}" +ENV FLASK_APP=script_runner.main:app +ENV SERVER_VERSION=$SERVER_VERSION + +ENTRYPOINT [ "/entrypoint.sh" ]