diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 3134387..1bd5d15 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -10,13 +10,63 @@ jobs: name: Publish script-runner to pypi runs-on: ubuntu-latest steps: - - name: Publish a Python distribution to PyPI + - name: Check out the repo + uses: actions/checkout@v2 + - name: Prepare + id: prep + run: | + PYTHON_IMAGE_TAG=alpine + DOCKER_IMAGE=labflow/script-runner + VERSION=latest + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:${PYTHON_IMAGE_TAG}-latest,${DOCKER_IMAGE}:${PYTHON_IMAGE_TAG}-${VERSION}" + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="$TAGS,${DOCKER_IMAGE}:${PYTHON_IMAGE_TAG}-sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=version::${VERSION} + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}" + echo ::set-output name=pyversion::"${PYTHON_IMAGE_TAG}" + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools pipenv + # pipenv lock --dev --requirements > requirements.dev.txt + # pip install -r requirements.txt + # - name: Lint with flake8 + # run: | + # # stop the build if there are Python syntax errors or undefined names + # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + # - name: Test with pytest + # run: | + # pytest + - name: Build python package + run: | + python setup.py sdist + env: + SCRIPT_RUNNER_VERSION: ${{ steps.prep.outputs.version }} + - name: Deploy to PyPI + if: success() && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} publish-docker: + needs: publish-pypi + if: startsWith(github.ref, 'refs/tags') name: Publish script-runner to dockerhub runs-on: ubuntu-latest steps: @@ -64,7 +114,7 @@ jobs: uses: docker/build-push-action@v2 with: context: ./docker - file: ./Dockerfile + file: ./docker/Dockerfile builder: ${{ steps.buildx.outputs.name }} push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.prep.outputs.tags }} @@ -78,10 +128,97 @@ jobs: SCRIPT_RUNNER_VERSION=${{ steps.prep.outputs.version }} PYTHON_IMAGE_TAG=${{ steps.prep.outputs.pyversion }} + publish-docker-example: + needs: publish-pypi + if: startsWith(github.ref, 'refs/tags') + name: Build and push script-runner-server-example docker image + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=labflow/script-runner-example + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}" + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=version::${VERSION} + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}" + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 + - name: Cache Docker layers + uses: actions/cache@v2.1.4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Push to Docker Hub + uses: docker/build-push-action@v2 + with: + context: ./docker + file: ./docker/Dockerfile.example + builder: ${{ steps.buildx.outputs.name }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.prep.outputs.tags }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.created=${{ steps.prep.outputs.created }} + org.opencontainers.image.revision=${{ github.sha }} + build-args: | + SERVER_VERSION=${{ steps.prep.outputs.hashtag }} + deploy-aws-example: + needs: publish-docker-example + if: startsWith(github.ref, 'refs/tags') name: Deploy swabseq-analysis-example to AWS runs-on: ubuntu-latest steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=labflow/script-runner-example + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}" + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=version::${VERSION} + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}" + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: @@ -92,28 +229,29 @@ jobs: - name: Setup Terraform tfvars uses: nowactions/envsubst@v1 with: - input: ./examples/aws/example.tfvars - output: ./examples/aws/terraform.tfvars + input: ./terraform/aws/example.tfvars + output: ./terraform/aws/terraform.tfvars env: DNS_SUBDOMAIN: ${{secrets.DNS_SUBDOMAIN}} DNS_ZONE_ID: ${{secrets.DNS_ZONE_ID}} + IMAGE_TAG: ${{steps.plan.outputs.hashtag}} - name: Setup Terraform uses: hashicorp/setup-terraform@v1 # - name: Terraform Format # id: fmt - # working-directory: ./examples/aws + # working-directory: ./terraform/aws # run: terraform fmt -check - name: Terraform Init id: init - working-directory: ./examples/aws + working-directory: ./terraform/aws run: terraform init - name: Terraform Plan id: plan - working-directory: ./examples/aws + working-directory: ./terraform/aws if: github.event_name == 'pull_request' run: terraform plan -no-color continue-on-error: true @@ -150,14 +288,40 @@ jobs: run: exit 1 - name: Terraform Apply - working-directory: ./examples/aws + working-directory: ./terraform/aws if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') run: terraform apply -auto-approve deploy-azure-example: + needs: publish-docker-example + if: startsWith(github.ref, 'refs/tags') name: Deploy swabseq-analysis-example to Azure runs-on: ubuntu-latest steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Prepare + id: prep + run: | + DOCKER_IMAGE=labflow/script-runner-example + VERSION=edge + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/} + elif [[ $GITHUB_REF == refs/heads/* ]]; then + VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') + elif [[ $GITHUB_REF == refs/pull/* ]]; then + VERSION=pr-${{ github.event.number }} + fi + TAGS="${DOCKER_IMAGE}:latest,${DOCKER_IMAGE}:${VERSION}" + if [ "${{ github.event_name }}" = "push" ]; then + TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" + fi + echo ::set-output name=version::${VERSION} + echo ::set-output name=tags::${TAGS} + echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo ::set-output name=hashtag::"sha-${GITHUB_SHA::8}" + - name: Configure Azure credentials uses: azure/login@v1 with: @@ -166,28 +330,29 @@ jobs: - name: Setup Terraform tfvars uses: nowactions/envsubst@v1 with: - input: ./examples/azure/example.tfvars - output: ./examples/azure/terraform.tfvars + input: ./terraform/azure/example.tfvars + output: ./terraform/azure/terraform.tfvars env: DNS_SUBDOMAIN: ${{secrets.DNS_SUBDOMAIN}} DNS_ZONE_NAME: ${{secrets.DNS_ZONE_NAME}} + IMAGE_TAG: ${{steps.plan.outputs.hashtag}} - name: Setup Terraform uses: hashicorp/setup-terraform@v1 # - name: Terraform Format # id: fmt - # working-directory: ./examples/azure + # working-directory: ./terraform/azure # run: terraform fmt -check - name: Terraform Init id: init - working-directory: ./examples/azure + working-directory: ./terraform/azure run: terraform init - name: Terraform Plan id: plan - working-directory: ./examples/azure + working-directory: ./terraform/azure if: github.event_name == 'pull_request' run: terraform plan -no-color continue-on-error: true @@ -224,6 +389,6 @@ jobs: run: exit 1 - name: Terraform Apply - working-directory: ./examples/azure + working-directory: ./terraform/azure if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') run: terraform apply -auto-approve diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..10af756 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,41 @@ +version: "3.7" +services: + server: + image: labflow/script-runner-example:latest + build: + dockerfile: ./docker/Dockerfile.example + context: . + args: + SERVER_VERSION: local+devcontainer + command: "python3 -m flask run --host=0.0.0.0 --port=5000" + environment: + - FLASK_ENV=development + - PORT=5000 + - PROPAGATE_EXCEPTIONS=True + - "SERVER_NAME=${SERVER_NAME}" + - "AUTH_PROVIDER=none" + - "CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379" + - "CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379" + ports: + - 5000:5000 + volumes: + - ./:/app + worker: + image: labflow/script-runner-example:latest + build: + context: . + dockerfile: ./docker/Dockerfile.example + args: + SERVER_VERSION: local+devcontainer + command: "python3 -m celery -A script_runner.analysis worker" + environment: + - "CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379" + - "CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379" + - "COMMAND_RUNDIR_BASE=/base-rundir" + volumes: + - ./:/app + redis: + image: redis:6-alpine + command: redis-server --requirepass ${REDIS_PASSWORD} + ports: + - 6379:6379 diff --git a/docker/Dockerfile b/docker/Dockerfile index bd28b6c..b7e521c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,18 +1,18 @@ -ARG SCRIPT_RUNNER_VERSION=local+container ARG PYTHON_IMAGE_TAG=alpine -FROM python:${PYTHON_IMAGE_TAG}} +FROM python:${PYTHON_IMAGE_TAG} + +ARG SCRIPT_RUNNER_VERSION=local+container WORKDIR /app -RUN pip install script-runner-api==${SCRIPT_RUNNER_VERSION} +RUN pip install \ + script-runner-api==${SCRIPT_RUNNER_VERSION} COPY ./entrypoint.sh /entrypoint.sh RUN dos2unix /entrypoint.sh RUN chmod +x /entrypoint.sh -COPY ./ . - ENV FLASK_APP=script_runner.main:app ENV SERVER_VERSION=$SERVER_VERSION diff --git a/docker/Dockerfile.example b/docker/Dockerfile.example new file mode 100644 index 0000000..098647a --- /dev/null +++ b/docker/Dockerfile.example @@ -0,0 +1,84 @@ +ARG SERVER_VERSION=local+container +ARG SCRIPT_RUNNER_VERSION=local+container + +# Start with rocker/tidyverse base image +FROM rocker/verse:3.6.3 + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# get and install bcl2fastq +RUN cd /tmp \ + && wget https://www.dropbox.com/s/idi0xfu0thurk7q/bcl2fastq2-v2-20-0-linux-x86-64.zip \ + && unzip bcl2fastq2-v2-20-0-linux-x86-64.zip -p \ + | tar xvz -C / usr/local/bin/bcl2fastq \ + && rm /tmp/bcl2fastq2* + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Install extra *nix utils +# x11, mesa, glu1 are so we can install paletteer +RUN apt-get update \ + && apt-get install -y \ + build-essential \ + libpq-dev \ + pigz \ + vim \ + git \ + less \ + curl \ + wget \ + parallel \ + python3-pip \ + bzip2 \ + libcairo2-dev \ + libfontconfig1-dev \ + ca-certificates \ + dos2unix \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# R Deps +RUN install2.r --error \ + BiocManager \ + seqinr \ + viridis \ + GGally \ + reader \ + plater \ + XML \ + DT \ + glmnet \ + speedglm \ + sandwich \ + ggbeeswarm \ + stringdist \ + argparser \ + kableExtra + +# Install bioconductor packages +RUN R --slave -e "BiocManager::install(c('savR', 'edgeR', 'qvalue', 'ShortRead', 'Rqc'))" + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# swabseq-analysis script +RUN git clone git://github.com/lab-grid/swabseq-analysis /app + +#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Python Env +WORKDIR /app + +RUN pip3 install \ + pandas \ + script-runner-api==${SCRIPT_RUNNER_VERSION} + +COPY ./entrypoint.sh /entrypoint.sh +RUN dos2unix /entrypoint.sh +RUN chmod +x /entrypoint.sh + +RUN mkdir /base-rundir +RUN wget -qO- https://swabseq-analysis-examples.s3-us-west-1.amazonaws.com/bcls/H3FY3K.tar.gz \ + | tar xvz -C /base-rundir + +ENV PYTHONPATH="${RBASE}:${PYTHONPATH}" +ENV FLASK_APP=script_runner.main:app +ENV SERVER_VERSION=$SERVER_VERSION + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/terraform/aws/example.tfvars b/terraform/aws/example.tfvars index bf6dc08..34d50d8 100644 --- a/terraform/aws/example.tfvars +++ b/terraform/aws/example.tfvars @@ -1,4 +1,4 @@ dns_subdomain = "${DNS_SUBDOMAIN}" dns_zone_id = "${DNS_ZONE_ID}" -image_tag = "sha-d59d449a" +image_tag = "${IMAGE_TAG}" diff --git a/terraform/aws/main.tf b/terraform/aws/main.tf index c6d096f..a7e45e5 100644 --- a/terraform/aws/main.tf +++ b/terraform/aws/main.tf @@ -4,7 +4,7 @@ provider "aws" { terraform { backend "s3" { - bucket = "swabseq-analysis-example-tf-backend" + bucket = "labflow-tf-backend" key = "swabseq-analysis-example-tf" region = "us-west-1" } diff --git a/terraform/azure/example.tfvars b/terraform/azure/example.tfvars index e123244..43949b3 100644 --- a/terraform/azure/example.tfvars +++ b/terraform/azure/example.tfvars @@ -1,4 +1,4 @@ dns_subdomain = "${DNS_SUBDOMAIN}" dns_zone_name = "${DNS_ZONE_NAME}" -image_tag = "sha-d59d449a" +image_tag = "${IMAGE_TAG}"