From fb47e1e7bea17a27b5a739664051ba7cee28f624 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 5 Mar 2022 19:26:19 -0500 Subject: [PATCH 01/26] updating app to use current GenomicDistributions API --- apps/LOLAweb/app.R | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/LOLAweb/app.R b/apps/LOLAweb/app.R index 5c93360..aced9e3 100644 --- a/apps/LOLAweb/app.R +++ b/apps/LOLAweb/app.R @@ -598,7 +598,7 @@ server <- function(input, output, session) { # calculate distribution over chromosomes for plotting if (paste0("chromSizes_", input$refgenome) %in% gd_data) { - genDist = aggregateOverGenomeBins(userSets, input$refgenome) + genDist = calcChromBinsRef(userSets, input$refgenome) } else { @@ -609,7 +609,7 @@ server <- function(input, output, session) { # calculate distances to TSSs if (paste0("TSS_", input$refgenome) %in% gd_data) { - TSSDist = TSSDistance(userSets, input$refgenome) + TSSDist = calcFeatureDistRefTSS(userSets, input$refgenome) } else { @@ -621,7 +621,7 @@ server <- function(input, output, session) { # distribution of overlaps for a query set to genomic partitions if (paste0("geneModels_", input$refgenome) %in% gd_data) { - gp = genomicPartitions(userSets, input$refgenome) + gp = calcPartitionsRef(userSets, input$refgenome) } else { @@ -1177,7 +1177,7 @@ server <- function(input, output, session) { } else { - plotGenomeAggregate(genomeAggregate = genDist) + + plotChromBins(genomeAggregate = genDist) + guides(fill=guide_legend(title="User set"), col = guide_legend(title="User set")) From b12a742ae680de47722ceb2e5bfc0e99e19f53a7 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sun, 6 Mar 2022 10:32:29 -0500 Subject: [PATCH 02/26] now using file.path instead of paste to more reliably construct dir paths in app --- apps/LOLAweb/misc.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/LOLAweb/misc.R b/apps/LOLAweb/misc.R index f3214ad..6a2407a 100644 --- a/apps/LOLAweb/misc.R +++ b/apps/LOLAweb/misc.R @@ -92,16 +92,16 @@ getEnv = function(envVar, default="") { # Set up cache dir based on environment variable localDir = getEnv("LWLOCAL", "/data/lola/") -cacheDir = paste0(localDir, "cache/") -logDir = paste0(localDir, "shinylog/") -resultsDir = paste0(localDir, "results/") +cacheDir = file.path(localDir, "cache/") +logDir = file.path(localDir, "shinylog/") +resultsDir = file.path(localDir, "results/") refDir = getEnv("LWREF", "/mnt/q/shefflab/LOLAweb/") -dbDir = paste0(refDir, "databases/") -universeDir = paste0(refDir, "universes/") -exampleDir = paste0(refDir, "examples/") +dbDir = file.path(refDir, "databases/") +universeDir = file.path(refDir, "universes/") +exampleDir = file.path(refDir, "examples/") message("Local dir: ", localDir) message("Cache dir: ", cacheDir) message("Reference data dir: ", refDir) From 2a6a9f2c59bfcde4a955d3b2272c8eb7de86ddf4 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 10:59:00 -0500 Subject: [PATCH 03/26] Initial stab at a GHA flow --- .github/workflows/build.yml | 63 +++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..675f44e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,63 @@ +name: Build & Deploy + +on: + push: + branches: + - 'master' + - 'main' + - 'dev' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - + name: Prints the current branch name + run: echo "${GITHUB_BRANCH##*/}" + env: + GH_BRANCH: ${{ github.ref }} + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to GHCR + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_PAT }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + push: true + tags: ghcr.io/databio/lolaweb:$GH_BRANCH + - + name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} + + deploy: + runs-on: ubuntu-latest + steps: + - + name: Prints the current branch name + run: echo "${GITHUB_BRANCH##*/}" + env: + GH_BRANCH: ${{ github.ref }} + - + name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.LOLA_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.LOLA_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - + name: Send SQS message + uses: isbang/sqs-action@v0.1.1 + with: + sqs-url: https://queue.amazonaws.com/474683445819/lola-updates + message: $GH_BRANCH From 3aa2f4d9df4fad607119ae78c277eda2eaa85651 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 11:13:04 -0500 Subject: [PATCH 04/26] Syntax for GHA variable --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 675f44e..70f1ec3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: ghcr.io/databio/lolaweb:$GH_BRANCH + tags: ghcr.io/databio/lolaweb:${GH_BRANCH} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} @@ -60,4 +60,4 @@ jobs: uses: isbang/sqs-action@v0.1.1 with: sqs-url: https://queue.amazonaws.com/474683445819/lola-updates - message: $GH_BRANCH + message: ${GH_BRANCH} From 61a18f0c76799362836a3e1f9e3ed5565f13a039 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 11:15:57 -0500 Subject: [PATCH 05/26] New action for BRANCH --- .github/workflows/build.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70f1ec3..cc10a12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,10 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - - name: Prints the current branch name - run: echo "${GITHUB_BRANCH##*/}" - env: - GH_BRANCH: ${{ github.ref }} + name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.1 - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -35,7 +34,7 @@ jobs: uses: docker/build-push-action@v2 with: push: true - tags: ghcr.io/databio/lolaweb:${GH_BRANCH} + tags: ghcr.io/databio/lolaweb:${{ steps.branch-name.outputs.current_branch }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} @@ -44,10 +43,9 @@ jobs: runs-on: ubuntu-latest steps: - - name: Prints the current branch name - run: echo "${GITHUB_BRANCH##*/}" - env: - GH_BRANCH: ${{ github.ref }} + name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v5.1 - name: Set up AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -60,4 +58,4 @@ jobs: uses: isbang/sqs-action@v0.1.1 with: sqs-url: https://queue.amazonaws.com/474683445819/lola-updates - message: ${GH_BRANCH} + message: ${{ steps.branch-name.outputs.current_branch }} From da54cb5fcac29b02307a4846a59c2ac6db9fbd43 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 11:25:54 -0500 Subject: [PATCH 06/26] Extending the build --- .github/workflows/build.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc10a12..0a3f1e9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,10 +11,6 @@ jobs: build: runs-on: ubuntu-latest steps: - - - name: Get branch name - id: branch-name - uses: tj-actions/branch-names@v5.1 - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -32,9 +28,16 @@ jobs: name: Build and push id: docker_build uses: docker/build-push-action@v2 - with: - push: true - tags: ghcr.io/databio/lolaweb:${{ steps.branch-name.outputs.current_branch }} + if: github.ref == 'refs/heads/master' + with: + file: docker/Dockerfile + push: true + tags: ghcr.io/databio/lolaweb:master + if: github.ref == 'refs/heads/dev' + with: + file: docker/dev/Dockerfile + push: true + tags: ghcr.io/databio/lolaweb:dev - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From a7dcdd5e7043933f7e9ab80148ef18596b98a29d Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 15:00:51 -0500 Subject: [PATCH 07/26] Working stub for Actions --- .github/workflows/build.yml | 57 +++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0a3f1e9..22a0d17 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,46 +1,47 @@ -name: Build & Deploy +name: Docker Image CI on: push: - branches: - - 'master' - - 'main' - - 'dev' + branches: [ master, dev ] + pull_request: + branches: [ master ] jobs: build: runs-on: ubuntu-latest steps: - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Login to GHCR + - name: Git checkout + uses: actions/checkout@v2 + + - name: Build DEV + if: github.ref == 'refs/heads/dev' + run: | + docker build -t ghcr.io/databio/lolaweb:dev docker/dev + + - name: Build MASTER + if: github.ref == 'refs/heads/master' + run: | + docker build -t ghcr.io/databio/lolaweb:latest docker + + - name: Login to GHCR uses: docker/login-action@v1 with: registry: ghcr.io username: ${{ secrets.GHCR_USERNAME }} password: ${{ secrets.GHCR_PAT }} - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v2 + + - name: Push :latest to GHCR + id: push_latest_ghcr if: github.ref == 'refs/heads/master' - with: - file: docker/Dockerfile - push: true - tags: ghcr.io/databio/lolaweb:master + run: docker push ghcr.io/databio/lolaweb:latest + + - name: Push :dev to GHCR + id: push_dev_ghcr if: github.ref == 'refs/heads/dev' - with: - file: docker/dev/Dockerfile - push: true - tags: ghcr.io/databio/lolaweb:dev - - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + run: docker push ghcr.io/databio/lolaweb:dev + + - name: Image digest + run: echo ${{ steps.push_ghcr.outputs.digest }} deploy: runs-on: ubuntu-latest From 6a69c2eb5abcc220bd8634da7acfbbb41954ecf9 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 15:04:45 -0500 Subject: [PATCH 08/26] Cleaner branch for SQS --- .github/workflows/build.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 22a0d17..329e1d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,11 +45,8 @@ jobs: deploy: runs-on: ubuntu-latest + needs: build steps: - - - name: Get branch name - id: branch-name - uses: tj-actions/branch-names@v5.1 - name: Set up AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -62,4 +59,4 @@ jobs: uses: isbang/sqs-action@v0.1.1 with: sqs-url: https://queue.amazonaws.com/474683445819/lola-updates - message: ${{ steps.branch-name.outputs.current_branch }} + message: ${GITHUB_REF##*/} From 90ca4f2fc66c6862699c6dd0a5cc5ba931f2f4f7 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Thu, 10 Mar 2022 18:56:57 -0500 Subject: [PATCH 09/26] updating dockerfiles with FROM ghcr shinybase; also updated maintainer email --- docker/Dockerfile | 4 ++-- docker/dev/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index de93945..bea6d0a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ -FROM databio/shinybase:latest +FROM ghcr.io/databio/shinybase:latest -MAINTAINER VP Nagraj "vpnagraj@virginia.edu" +MAINTAINER VP Nagraj "nagraj@nagraj.net" # move conf files COPY shiny-server.conf /etc/shiny-server/shiny-server.conf diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 718b84d..88e9351 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,6 +1,6 @@ -FROM databio/shinybase:latest +FROM ghcr.io/databio/shinybase:latest -MAINTAINER VP Nagraj "vpnagraj@virginia.edu" +MAINTAINER VP Nagraj "nagraj@nagraj.net" # move conf files COPY shiny-server.conf /etc/shiny-server/shiny-server.conf From 7ec9d3846d249b9fb4af0fc33ff7983150cb4a64 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Thu, 10 Mar 2022 21:37:44 -0500 Subject: [PATCH 10/26] Testing build with new shinybase --- docker/dev/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index 718b84d..31ea0d7 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,4 +1,5 @@ -FROM databio/shinybase:latest +FROM ghcr.io/databio/shinybase:latest +# FROM databio/shinybase:latest MAINTAINER VP Nagraj "vpnagraj@virginia.edu" From 5ae69e8dcd21efe1c022501523fd32e4f3c4ea22 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Fri, 11 Mar 2022 08:24:10 -0500 Subject: [PATCH 11/26] Shifting referencees to GHCR - in docs and Dockerfiles --- docker/Dockerfile | 6 +++--- docker/README.md | 16 ++++++++-------- docker/swarm/README.md | 18 +++++++++--------- docker/swarm/docker-compose.yml | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index de93945..306f7a1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,10 +1,10 @@ -FROM databio/shinybase:latest +FROM ghcr.io/databio/shinybase:latest MAINTAINER VP Nagraj "vpnagraj@virginia.edu" # move conf files -COPY shiny-server.conf /etc/shiny-server/shiny-server.conf -COPY shiny-server.sh /usr/bin/shiny-server.sh +COPY ./shiny-server.conf /etc/shiny-server/shiny-server.conf +COPY ./shiny-server.sh /usr/bin/shiny-server.sh # get the code WORKDIR /srv/shiny-server diff --git a/docker/README.md b/docker/README.md index cbf83a0..9831ccf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,8 +1,9 @@ -[![Docker pulls](https://img.shields.io/docker/pulls/databio/lolaweb.svg)](https://hub.docker.com/r/databio/lolaweb/) [![Build Status](https://travis-ci.org/databio/LOLAweb.svg?branch=master)](https://travis-ci.org/databio/LOLAweb) [![](https://images.microbadger.com/badges/image/databio/lolaweb.svg)](https://microbadger.com/images/databio/lolaweb "Get your own image badge on microbadger.com") +[![Docker Image CI](https://github.com/databio/LOLAweb/actions/workflows/build.yml/badge.svg?branch=dev)](https://github.com/databio/LOLAweb/actions/workflows/build.yml) + # Shiny LOLAweb for Docker -The `databio/lolaweb` container is based on the `databio/shinybase` container, which you can find in its [github repository](https://github.com/databio/shinyBase) or [on dockerhub](https://hub.docker.com/r/databio/shinybase/). +The `ghcr.io/databio/lolaweb` container is based on the `ghcr.io/databio/shinybase` container, which you can find in its [github repository](https://github.com/databio/shinyBase) or in the [GitHub Container Registry](https://github.com/databio/shinyBase/pkgs/container/shinybase). ## `build` the container image yourself @@ -14,10 +15,9 @@ The `databio/lolaweb` container is based on the `databio/shinybase` container, w ## Or `pull` the container image: -```docker pull databio/lolaweb``` - -The image itself is hosted at dockerhub: https://hub.docker.com/r/databio/lolaweb/ +```docker pull ghcr.io/databio/lolaweb``` +The container image itself is hosted in the GitHub Container Registry: https://github.com/databio/LOLAweb/pkgs/container/lolaweb ## Container volumes and reference data @@ -54,16 +54,16 @@ LOLAweb looks for `$LWLOCAL` to have two subfolders: `cache`, and `shinylogs`. T --volume ${LWLOCAL}:${LWLOCAL} \ --volume ${LWREF}:${LWREF} \ --volume ${LWLOCAL}/shinylog:/var/log/shiny-server - databio/lolaweb + ghcr.io/databio/lolaweb Open a browser to: ``` http://localhost/LOLAweb/apps/LOLAweb ``` -## Running a dev container +## Running a `dev` container -You could also run the `dev` version of the container by pulling `databio/lolaweb:dev`. This will retrieve the dev tagged image from dockerhub. Just add `:dev` to the container name at the end of the `docker run` command above. +You could also run the `dev` version of the container by pulling `ghcr.io/databio/lolaweb:dev`. This will retrieve the dev tagged image from the GitHub Container Registry. Just add `:dev` to the container name at the end of the `docker run` command above. ## Running multiple LOLAweb containers simultaneously with Docker Swarm: diff --git a/docker/swarm/README.md b/docker/swarm/README.md index c87f0da..e960672 100644 --- a/docker/swarm/README.md +++ b/docker/swarm/README.md @@ -31,7 +31,7 @@ version: "3" services: production: - image: databio/lolaweb:latest + image: ghcr.io/databio/lolaweb:latest networks: - net ports: @@ -96,12 +96,12 @@ Take note of a few important elements of this YAML file: - Træfik mounts and monitors the Docker socket file in order to automatically update anytime the swarm behind it is updated. ## Container Images -Images used in this stack must be downloadable as pre-built images from a container repository such as [Docker Hub](https://hub.docker.com/). +Images used in this stack must be downloadable as pre-built images from a container repository such as [Docker Hub](https://hub.docker.com/) or the GitHub Container Registry. Hand-built local images will fail to deploy unless you push them to a repository (such as Docker Hub) and then pull from there. Pull the LOLAweb and Træfik container images: - $ docker pull databio/lolaweb + $ docker pull ghcr.io/databio/lolaweb $ docker pull traefik @@ -153,9 +153,9 @@ First, list the services in your stack: $ docker service ls - ID NAME MODE REPLICAS IMAGE PORTS - d7lzimlt0ayj lola_loadbal replicated 1/1 traefik:latest *:80->80/tcp,*:3939->8080/tcp - 572dn5lstoh0 lola_app replicated 4/4 databio/lolaweb:latest *:30003->80/tcp + ID NAME MODE REPLICAS IMAGE PORTS + d7lzimlt0ayj lola_loadbal replicated 1/1 traefik:latest *:80->80/tcp,*:3939->8080/tcp + 572dn5lstoh0 lola_app replicated 4/4 ghcr.io/databio/lolaweb:latest *:30003->80/tcp If necessary, inspect the specific service you want to update (by name): @@ -165,12 +165,12 @@ If necessary, inspect the specific service you want to update (by name): Next, pull the newer container image(s): - $ docker pull databio/lolaweb:latest + $ docker pull ghcr.io/databio/lolaweb:latest Finally, update the service to reference the newest version of the container: - $ docker service update --image databio/lolaweb:latest lola_app + $ docker service update --image ghcr.io/databio/lolaweb:latest lola_app If you change parameters of the `docker-compose.yml` file, just run the `docker stack deploy` command again to refresh: @@ -214,7 +214,7 @@ minutes to complete. The workflow steps are: 1. Developers push code changes back to a specific branch of the project in GitHub. -2. Travis-CI is plugged in with steps defined in `.travis.yml`. Travis builds the appropriate container, based on Dockerfiles specific to each branch. It then pushes the container to Docker Hub, and sends a simple SQS message to a queue with the name of the updated branch. +2. Travis-CI is plugged in with steps defined in `.travis.yml`. Travis builds the appropriate container, based on Dockerfiles specific to each branch. It then pushes the container to GHCR, and sends a simple SQS message to a queue with the name of the updated branch. 3. Finally, the LOLAweb servers use a cron job that runs long polling requests to SQS, waiting for a message. When one arrives, the Docker service is updated with the new container image. ## More Information diff --git a/docker/swarm/docker-compose.yml b/docker/swarm/docker-compose.yml index ed4a3d1..8cb96e3 100644 --- a/docker/swarm/docker-compose.yml +++ b/docker/swarm/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: production: - image: databio/lolaweb:latest + image: ghcr.io/databio/lolaweb:latest networks: - net ports: From e664b0ec0defe74843549e97e843af6e987d1877 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Fri, 11 Mar 2022 08:29:28 -0500 Subject: [PATCH 12/26] Fix Travis references in docs --- docker/swarm/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/swarm/README.md b/docker/swarm/README.md index e960672..9d13d3b 100644 --- a/docker/swarm/README.md +++ b/docker/swarm/README.md @@ -207,14 +207,14 @@ To remove a service: ## Continuous Integration To ease the deployment of production and dev containers for this project, we make use of a continuous -integration workflow using Travis-CI and Amazon SQS. This automates the build and deployment steps after +integration workflow using GitHub Actions and Amazon SQS. This automates the build and deployment steps after code changes are committed and pushed back to GitHub. Automated builds and deployments take approximately 10-15 minutes to complete. The workflow steps are: 1. Developers push code changes back to a specific branch of the project in GitHub. -2. Travis-CI is plugged in with steps defined in `.travis.yml`. Travis builds the appropriate container, based on Dockerfiles specific to each branch. It then pushes the container to GHCR, and sends a simple SQS message to a queue with the name of the updated branch. +2. GitHub Actions are triggered to execute steps defined in `.github/workflows/build.yml`. GHA builds the appropriate container, based on Dockerfiles specific to each branch. It then pushes the container image to GHCR, and sends a simple SQS message to a queue with the name of the updated branch. 3. Finally, the LOLAweb servers use a cron job that runs long polling requests to SQS, waiting for a message. When one arrives, the Docker service is updated with the new container image. ## More Information @@ -225,6 +225,6 @@ The workflow steps are: - [Compose Documentation](https://docs.docker.com/compose/reference/) - [Swarm Documentation](https://docs.docker.com/get-started/part4/) - [Træfik Load Balancer](https://docs.traefik.io/) -- [Travis Continuous Integration](https://docs.travis-ci.com/) +- [GitHub Actions Continuous Integration](https://lab.github.com/githubtraining/github-actions:-continuous-integration) This solution is based on the [slopp/Load-Test](https://github.com/slopp/Load-Test/) documentation in GitHub. From af9123327bbe675916f7d6c41c34a66eff3e2814 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 11:55:26 -0500 Subject: [PATCH 13/26] re-org of files to simplify docker build; adding v number to app via version.txt --- docker/Dockerfile => Dockerfile | 4 +-- docker/README.md => README-docker.md | 0 apps/LOLAweb/app.R | 14 +++++---- apps/LOLAweb/version.txt | 1 + docker/dev/Dockerfile | 22 -------------- docker/dev/shiny-server.conf | 29 ------------------- docker/shiny-server.sh | 14 --------- docker/shiny-server.conf => shiny-server.conf | 0 docker/dev/shiny-server.sh => shiny-server.sh | 0 {docker/swarm => swarm}/README.md | 0 {docker/swarm => swarm}/docker-compose.yml | 0 11 files changed, 12 insertions(+), 72 deletions(-) rename docker/Dockerfile => Dockerfile (85%) rename docker/README.md => README-docker.md (100%) create mode 100644 apps/LOLAweb/version.txt delete mode 100644 docker/dev/Dockerfile delete mode 100644 docker/dev/shiny-server.conf delete mode 100755 docker/shiny-server.sh rename docker/shiny-server.conf => shiny-server.conf (100%) rename docker/dev/shiny-server.sh => shiny-server.sh (100%) rename {docker/swarm => swarm}/README.md (100%) rename {docker/swarm => swarm}/docker-compose.yml (100%) diff --git a/docker/Dockerfile b/Dockerfile similarity index 85% rename from docker/Dockerfile rename to Dockerfile index bea6d0a..a156cea 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ COPY shiny-server.sh /usr/bin/shiny-server.sh # get the code WORKDIR /srv/shiny-server -RUN git clone https://github.com/databio/LOLAweb.git +COPY apps/LOLAweb /srv/shiny-server/LOLAweb/apps/LOLAweb # add dir for cache RUN mkdir LOLAweb/cache @@ -18,5 +18,5 @@ RUN chown -R shiny:shiny LOLAweb/cache RUN mkdir LOLAweb/apps/LOLAweb/plots RUN chown -R shiny:shiny LOLAweb/apps/LOLAweb/plots -# run the server setup script +## run the server setup script CMD ["/usr/bin/shiny-server.sh"] diff --git a/docker/README.md b/README-docker.md similarity index 100% rename from docker/README.md rename to README-docker.md diff --git a/apps/LOLAweb/app.R b/apps/LOLAweb/app.R index aced9e3..0dac57a 100644 --- a/apps/LOLAweb/app.R +++ b/apps/LOLAweb/app.R @@ -17,8 +17,12 @@ library(plotly) setCacheDir(cacheDir) # get lolaweb version -lw_version <- system(command = "git rev-parse HEAD | cut -c1-9", intern = TRUE) - +#lw_version <- system(command = "git rev-parse HEAD | cut -c1-9", intern = TRUE) +if(file.exists("version.txt")) { + lw_version <- readLines("version.txt") +} else { + lw_version <- NA +} ui <- list( # # need empty fluid page at top with height 0 to preserve window title @@ -229,9 +233,9 @@ ui <- list( target = 'blank'>Sheffield Computational Biology Lab and SOMRC at UVA.
View source code on GitHub or run it locally with our docker image", "
LOLAweb version: ", lw_version, + ='https://github.com/databio/LOLAweb/releases/tag/", lw_version, "' target='blank'>", lw_version, "") ), align = "center", style = " bottom:0; width:100%; height:10px; padding: 10px; padding-bottom:20px; z-index: 1000;"), @@ -900,7 +904,7 @@ server <- function(input, output, session) { "Genome ", "Universe ", "Database ", - "LOLAweb commit used "), + "LOLAweb version used "), y = c(as.character(rawdat_res$run_sum$start_time), as.character(rawdat_res$run_sum$end_time), diff --git a/apps/LOLAweb/version.txt b/apps/LOLAweb/version.txt new file mode 100644 index 0000000..0d0c52f --- /dev/null +++ b/apps/LOLAweb/version.txt @@ -0,0 +1 @@ +v1.4.0 diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile deleted file mode 100644 index 88e9351..0000000 --- a/docker/dev/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM ghcr.io/databio/shinybase:latest - -MAINTAINER VP Nagraj "nagraj@nagraj.net" - -# move conf files -COPY shiny-server.conf /etc/shiny-server/shiny-server.conf -COPY shiny-server.sh /usr/bin/shiny-server.sh - -# get the code -WORKDIR /srv/shiny-server -RUN git clone -b dev https://github.com/databio/LOLAweb.git - -# add dir for cache -RUN mkdir LOLAweb/cache -RUN chown -R shiny:shiny LOLAweb/cache - -# add plots dir for zipping up all figures to download with one button -RUN mkdir LOLAweb/apps/LOLAweb/plots -RUN chown -R shiny:shiny LOLAweb/apps/LOLAweb/plots - -# run the server setup script -CMD ["/usr/bin/shiny-server.sh"] diff --git a/docker/dev/shiny-server.conf b/docker/dev/shiny-server.conf deleted file mode 100644 index 3f7e026..0000000 --- a/docker/dev/shiny-server.conf +++ /dev/null @@ -1,29 +0,0 @@ -# Define the user we should use when spawning R Shiny processes -run_as shiny; - -# disable websocket -disable_protocols websocket; - -# disable idle timeout to retain connection -app_idle_timeout 0; - -# Define a top-level server which will listen on a port -server { - # Instruct this server to listen on port 80. The app at dokku-alt need expose PORT 80, or 500 e etc. See the docs - listen 80; - - # Define the location available at the base URL - location / { - - # Run this location in 'site_dir' mode, which hosts the entire directory - # tree at '/srv/shiny-server' - site_dir /srv/shiny-server; - - # Define where we should put the log files for this location - log_dir /var/log/shiny-server; - - # Should we list the contents of a (non-Shiny-App) directory when the user - # visits the corresponding URL? - directory_index on; - } -} diff --git a/docker/shiny-server.sh b/docker/shiny-server.sh deleted file mode 100755 index 2a2c51f..0000000 --- a/docker/shiny-server.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - - -# Make sure the directory for individual app logs exists -mkdir -p /var/log/shiny-server -chown shiny.shiny /var/log/shiny-server - -# make sure relevant environment variables are visible to shiny server .Renviron -env | grep "LWLOCAL\|LWREF" > /home/shiny/.Renviron -chown shiny.shiny /home/shiny/.Renviron - -exec shiny-server 2>&1 - -# done diff --git a/docker/shiny-server.conf b/shiny-server.conf similarity index 100% rename from docker/shiny-server.conf rename to shiny-server.conf diff --git a/docker/dev/shiny-server.sh b/shiny-server.sh similarity index 100% rename from docker/dev/shiny-server.sh rename to shiny-server.sh diff --git a/docker/swarm/README.md b/swarm/README.md similarity index 100% rename from docker/swarm/README.md rename to swarm/README.md diff --git a/docker/swarm/docker-compose.yml b/swarm/docker-compose.yml similarity index 100% rename from docker/swarm/docker-compose.yml rename to swarm/docker-compose.yml From 506a19ea196d4b42ffb385f21a2499b6a5d6d787 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 11:59:08 -0500 Subject: [PATCH 14/26] adding link to manuscript in about page --- apps/LOLAweb/about.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/LOLAweb/about.md b/apps/LOLAweb/about.md index 10616e4..717582e 100644 --- a/apps/LOLAweb/about.md +++ b/apps/LOLAweb/about.md @@ -340,8 +340,8 @@ Here are some links to other resources related to LOLA and LOLAweb: - [LOLAweb issue tracker](https://github.com/databio/LOLAweb/issues) - please use this for support requests for LOLAweb. - [LOLAweb GitHub source code](https://github.com/databio/LOLAweb) -- [Dockerhub docker image](https://hub.docker.com/r/databio/lolaweb/) -- Manuscript (pending) +- [Docker image](https://hub.docker.com/r/databio/lolaweb/) +- [Manuscript](https://doi.org/10.1093/nar/gky464) - [LOLA R package at Bioconductor](http://bioconductor.org/packages/LOLA/) - [LOLA R package source code at GitHub](https://github.com/nsheff/LOLA) - [LOLA R package documentation](http://code.databio.org/LOLA) From c796f4bf4b8f23363fa16a6aa9b29cfc95193975 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:26:12 -0500 Subject: [PATCH 15/26] consolidating docs throughout --- README-docker.md | 71 ------------------------ README.md | 123 ++++++++++++++++++++++++++++++++++++++++- apps/LOLAweb/README.md | 37 +------------ 3 files changed, 124 insertions(+), 107 deletions(-) delete mode 100644 README-docker.md diff --git a/README-docker.md b/README-docker.md deleted file mode 100644 index 9831ccf..0000000 --- a/README-docker.md +++ /dev/null @@ -1,71 +0,0 @@ -[![Docker Image CI](https://github.com/databio/LOLAweb/actions/workflows/build.yml/badge.svg?branch=dev)](https://github.com/databio/LOLAweb/actions/workflows/build.yml) - - -# Shiny LOLAweb for Docker - -The `ghcr.io/databio/lolaweb` container is based on the `ghcr.io/databio/shinybase` container, which you can find in its [github repository](https://github.com/databio/shinyBase) or in the [GitHub Container Registry](https://github.com/databio/shinyBase/pkgs/container/shinybase). - -## `build` the container image yourself - -1. Clone this repository -2. Build locally using Docker. Run this command from the same directory as the `Dockerfile`. - -```docker build --no-cache -t lolaweb .``` - - -## Or `pull` the container image: - -```docker pull ghcr.io/databio/lolaweb``` - -The container image itself is hosted in the GitHub Container Registry: https://github.com/databio/LOLAweb/pkgs/container/lolaweb - -## Container volumes and reference data - -LOLAweb needs access to a few folders where it can store results or logs, or access necessary files like the database. To handle this, we've set up the app to look for two shell environment variables: - -* **$LWREF**, for LOLAweb reference data, which may be read-only -* **$LWLOCAL**, where local results can be written. - -To run the LOLAweb container (locally or on a server), you need to set these environment variables (for example, in your `.bashrc` or `.profile` file. These variables will be injected into the container when it is run. - -For example, set these variables to two different paths if you like. Or if you keep all five subfolders together in the same path, set these variables to the same value. - -``` -# Example locations. Set to match your environment -export LREF='/home/subdir/lola' -export LWLOCAL='/var/things/loladata' -``` - - -### LWREF - -LOLAweb will look at the value in `$LWREF` for the reference data. This folder should have subfolders called `databases`, `universes`, and `examples`. In each of these subfolders are another layer of subfolders for genome assemblies. See the [LOLAweb documentation](https://github.com/databio/LOLAweb/tree/master/apps/LOLAweb) for downloading reference data. - -### LWLOCAL - -LOLAweb looks for `$LWLOCAL` to have two subfolders: `cache`, and `shinylogs`. This is where the app will write results and log files. If running LOLAweb on a server, be sure these directories are writeable by the Docker process. - -## Run the LOLAweb container locally with reference data: - - docker run -d \ - -p 80:80 \ - -e LWREF=$LWREF \ - -e LWLOCAL=$LWLOCAL \ - --volume ${LWLOCAL}:${LWLOCAL} \ - --volume ${LWREF}:${LWREF} \ - --volume ${LWLOCAL}/shinylog:/var/log/shiny-server - ghcr.io/databio/lolaweb - -Open a browser to: -``` -http://localhost/LOLAweb/apps/LOLAweb -``` - -## Running a `dev` container - -You could also run the `dev` version of the container by pulling `ghcr.io/databio/lolaweb:dev`. This will retrieve the dev tagged image from the GitHub Container Registry. Just add `:dev` to the container name at the end of the `docker run` command above. - - -## Running multiple LOLAweb containers simultaneously with Docker Swarm: - -For the typical use case of an individual user, a single running container will suffice. But if you need to set up an enterprise-level LOLAweb server that can handle concurrent users, we've also made that easy by using docker swarm. This is how we run the main LOLAweb servers, and you could do the same thing if you want your own local implementation. Docker Swarm is a technique for running multiple instances of the same container. [Read more](swarm/README.md) about how to set up your own swarm. diff --git a/README.md b/README.md index 167053f..bd07523 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ # LOLAweb + +[![Docker Image CI](https://github.com/databio/LOLAweb/actions/workflows/build.yml/badge.svg?branch=dev)](https://github.com/databio/LOLAweb/actions/workflows/build.yml) + LOLAweb is a web server and interactive results viewer for enrichment of overlap between a query region set (a bed file) and a database of region sets. It provides an interactive result explorer to visualize the highest ranked enrichments from the database. You can access the web server at . -This repository contains two components: 1) the shiny [app source code](apps/LOLAweb/) and 2) [Docker implementation](docker/) for LOLAweb. +This repository contain the shiny [app source code](apps/LOLAweb/) and Docker implementation for LOLAweb. ## Shiny app @@ -9,4 +12,120 @@ LOLAweb is implemented as an interactive Shiny app. You can run this app locally ## Docker -We have also produced a Dockerfile and published an image on Docker Hub (`databio/lolaweb`). Learn how to run LOLAweb locally in a docker container by reading the [instructions in the docker folder](docker/). +The `ghcr.io/databio/lolaweb` container is based on the `ghcr.io/databio/shinybase` container, which you can find in its [github repository](https://github.com/databio/shinyBase) or in the [GitHub Container Registry](https://github.com/databio/shinyBase/pkgs/container/shinybase). + +### `build` the container image yourself + +1. Clone this repository +2. Build locally using Docker. Run this command from the same directory as the `Dockerfile`. + +```docker build --no-cache -t lolaweb .``` + + +### Or `pull` the container image: + +```docker pull ghcr.io/databio/lolaweb``` + +The container image itself is hosted in the GitHub Container Registry: https://github.com/databio/LOLAweb/pkgs/container/lolaweb + +### Container volumes and reference data + +LOLAweb needs access to a few folders where it can store results or logs, or access necessary files like the database. To handle this, we've set up the app to look for two shell environment variables: + +* **$LWREF**, for LOLAweb reference data, which may be read-only +* **$LWLOCAL**, where local results can be written. + +To run the LOLAweb container (locally or on a server), you need to set these environment variables (for example, in your `.bashrc` or `.profile` file. These variables will be injected into the container when it is run. + +For example, set these variables to two different paths if you like. Or if you keep all five subfolders together in the same path, set these variables to the same value. + +``` +# Example locations. Set to match your environment +export LWREF='/home/subdir/lola/' +export LWLOCAL='/var/things/loladata/' +``` + +LOLAweb will look at the value in `$LWREF` for the reference data. This folder should have subfolders called `databases`, `universes`, and `examples`. In each of these subfolders are another layer of subfolders for genome assemblies + +LOLAweb looks for `$LWLOCAL` to have two subfolders: `cache`, and `shinylogs`. This is where the app will write results and log files. If running LOLAweb on a server, be sure these directories are writeable by the Docker process. + +The following instructions demonstrate a minimal example of how to download and configure the LOLAweb data directories for a `hg19` reference data: + +``` +## assign env vars for data path +## NOTE: must include trailing / +LWLOCAL="/path/to/local/data/" +LWREF="/path/to/reference/data/" + +## change reference data dir +cd $LWREF + +## create dir for databases +mkdir -p databases +## create examples and universe dir +## NOTE: these must include subdirs named corresponding to appropriate ref genome +mkdir -p examples/hg19 +mkdir -p universes/hg19 + +## download example universe and user set +curl http://cloud.databio.org.s3.amazonaws.com/vignettes/lola_vignette_data_150505.tgz | tar xvz + +## move example universe and user set files to hg19 dir +mv lola_vignette_data/activeDHS_universe.bed universes/hg19/. +mv lola_vignette_data/setB_100.bed examples/hg19/. + +## clean up +rm -rf lola_vignette_data + +## download databases +curl http://cloud.databio.org.s3.amazonaws.com/regiondb/LOLACoreCaches_170206.tgz | tar xvz +curl http://cloud.databio.org.s3.amazonaws.com/regiondb/LOLAExtCaches_170206.tgz | tar xvz + +## move databases to appropriate spots +mv scratch/ns5bc/resources/regions/LOLACore databases/Core +mv scratch/ns5bc/resources/regions/LOLAExt databases/Extended + +## clean up +rm -rf scratch + +## change ot local data dir +cd $LWLOCAL + +## create placeholder dirs for cache and shinylog +mkdir -p cache +mkdir -p shinylog +``` + +### Run the LOLAweb container locally with reference data: + +``` +## run the docker image +docker run -d \ + -p 80:80 \ + -e LWREF=$LWREF \ + -e LWLOCAL=$LWLOCAL \ + --volume ${LWLOCAL}:${LWLOCAL} \ + --volume ${LWREF}:${LWREF} \ + --volume ${LWLOCAL}/shinylog:/var/log/shiny-server \ + databio/lolaweb +``` + +Open a browser to: +``` +http://localhost/LOLAweb/apps/LOLAweb +``` + +### Running a `dev` container + +You could also run the `dev` version of the container by pulling `ghcr.io/databio/lolaweb:dev`. This will retrieve the dev tagged image from the GitHub Container Registry. Just add `:dev` to the container name at the end of the `docker run` command above. + +### Running multiple LOLAweb containers simultaneously with Docker Swarm: + +For the typical use case of an individual user, a single running container will suffice. But if you need to set up an enterprise-level LOLAweb server that can handle concurrent users, we've also made that easy by using docker swarm. This is how we run the main LOLAweb servers, and you could do the same thing if you want your own local implementation. Docker Swarm is a technique for running multiple instances of the same container. [Read more](swarm/README.md) about how to set up your own swarm. + +### Troubleshooting + +The LOLAweb Docker implementation includes a mechanism to write Shiny Server logs to `$LWLOCAL/shinylog`. These log files may be useful when troubleshooting problems with running LOLAweb via Docker. They include errors with R processing as well as information as to whether the Shiny Server process was killed due to resource limitations (i.e., not enough RAM allocated to Docker daemon). + +For additional support with the LOLAweb Docker implementation, please file a [GitHub issue](https://github.com/databio/LOLAweb/issues). + diff --git a/apps/LOLAweb/README.md b/apps/LOLAweb/README.md index 7502c3d..a430d2a 100644 --- a/apps/LOLAweb/README.md +++ b/apps/LOLAweb/README.md @@ -25,39 +25,8 @@ source("https://bioconductor.org/biocLite.R") biocLite(c("LOLA", "GenomicRanges")) ``` -You'll also need underlying data that is not available in this repository in order to establish the universes, example user sets and reference genome directories. Run the following from within the root of this folder to download the data and create the `universes/`, `userSets/`, `reference/` and `cache/` directories respectively: +You'll also need underlying data that is not available in this repository in order to establish the universes, example user sets and reference genome directories. Detailed guidance for how to download and organize reference data is available in the [LOLAweb project README](https://github.com/databio/lolaweb/#readme). -``` -# create universes dir -mkdir universes -mkdir userSets - -# example universe and user set -curl http://cloud.databio.org.s3.amazonaws.com/vignettes/lola_vignette_data_150505.tgz | tar xvz - -mv lola_vignette_data/activeDHS_universe.bed universes/. -mv lola_vignette_data/setB_100.bed userSets/. - -rm -rf lola_vignette_data -``` +Note that one additional step for running the app locally willbe to either 1) pass the `$LWREF` and `$LWLOCAL` environment variables to your R session or 2) overwite the definition of `localDir` and `refDir` in `misc.R` with the respective paths on your machine. -``` -# create reference dir -mkdir reference - -# core -curl http://cloud.databio.org.s3.amazonaws.com/regiondb/LOLACoreCaches_170206.tgz | tar xvz -mv scratch/ns5bc/resources/regions/LOLACore reference/Core -rm -rf scratch - -# extended -curl http://cloud.databio.org.s3.amazonaws.com/regiondb/LOLAExtCaches_170206.tgz | tar xvz -mv scratch/ns5bc/resources/regions/LOLAExt reference/Extended -rm -rf scratch -``` - -``` -# create cache dir -mkdir cache -``` -With all of the above installed you can now launch the app with `shiny::runApp()` from within R at the root of this directory. \ No newline at end of file +With dependencies installed and data configuration complete you can now launch the app with `shiny::runApp()` from within R at the root of this directory. \ No newline at end of file From b7902336d2d735d321216ced47f4a90cc4aa7fe2 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:27:24 -0500 Subject: [PATCH 16/26] removing travis yml since we are now using GH actions --- .travis.yml | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6cd267b..0000000 --- a/.travis.yml +++ /dev/null @@ -1,43 +0,0 @@ -branches: - only: - - master - - staging - - dev - -language: bash - -services: - - docker - -before_install: - - sudo pip install --upgrade pip - - pip install --user awscli - - export PATH=$PATH:$HOME/.local/bin - - echo $DOCKER_PASSWORD | docker login -u "$DOCKER_USERNAME" --password-stdin - -install: - - if [ "$TRAVIS_BRANCH" == "master" ]; then - cd docker/ ; - docker build --no-cache -t databio/lolaweb:latest . || exit 1; - docker push databio/lolaweb:latest || exit 1; - aws sqs send-message --queue-url 'https://queue.amazonaws.com/474683445819/lola-updates' --message-body $TRAVIS_BRANCH || exit 1; - fi - - if [ "$TRAVIS_BRANCH" == "staging" ]; then - cd docker/staging/ ; - docker build --no-cache -t databio/lolaweb:staging . || exit 1; - docker push databio/lolaweb:staging || exit 1; - aws sqs send-message --queue-url 'https://queue.amazonaws.com/474683445819/lola-updates' --message-body $TRAVIS_BRANCH || exit 1; - fi - - if [ "$TRAVIS_BRANCH" == "dev" ]; then - cd docker/dev/ ; - docker build --no-cache -t databio/lolaweb:dev . || exit 1; - docker push databio/lolaweb:dev || exit 1; - aws sqs send-message --queue-url 'https://queue.amazonaws.com/474683445819/lola-updates' --message-body $TRAVIS_BRANCH || exit 1; - fi - -notifications: - email: - on_success: change - on_failure: always - recipients: - - lola@virginia.edu From 4eb69284c3264e150aaf2042bc4cb82ed7c02d30 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:33:56 -0500 Subject: [PATCH 17/26] updating sample result cache link --- apps/LOLAweb/app.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/LOLAweb/app.R b/apps/LOLAweb/app.R index 0dac57a..5fdae2c 100644 --- a/apps/LOLAweb/app.R +++ b/apps/LOLAweb/app.R @@ -105,7 +105,7 @@ ui <- list( actionButton("run", "RUN LOLA", class = "runLOLA"), - HTML("") + HTML("") ), id = "runInputs"), From a604d28dbd7fd341060f37e4a9b9d424c293946e Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:42:25 -0500 Subject: [PATCH 18/26] updating image name in example docker run --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bd07523..e882bcd 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ mkdir -p shinylog ``` ## run the docker image +## NOTE: this run command uses image pulled from ghcr.io/databio/lolaweb docker run -d \ -p 80:80 \ -e LWREF=$LWREF \ @@ -107,7 +108,7 @@ docker run -d \ --volume ${LWLOCAL}:${LWLOCAL} \ --volume ${LWREF}:${LWREF} \ --volume ${LWLOCAL}/shinylog:/var/log/shiny-server \ - databio/lolaweb + ghcr.io/databio/lolaweb ``` Open a browser to: From 8863b8fb35a23827d9a90b568a1d5915959d47ea Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:43:22 -0500 Subject: [PATCH 19/26] github to GitHub in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e882bcd..cb58529 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ LOLAweb is implemented as an interactive Shiny app. You can run this app locally ## Docker -The `ghcr.io/databio/lolaweb` container is based on the `ghcr.io/databio/shinybase` container, which you can find in its [github repository](https://github.com/databio/shinyBase) or in the [GitHub Container Registry](https://github.com/databio/shinyBase/pkgs/container/shinybase). +The `ghcr.io/databio/lolaweb` container is based on the `ghcr.io/databio/shinybase` container, which you can find in its [GitHub repository](https://github.com/databio/shinyBase) or in the [GitHub Container Registry](https://github.com/databio/shinyBase/pkgs/container/shinybase). ### `build` the container image yourself From fc248dc87c12a2a284df32349d7a2ba0990c9189 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:46:55 -0500 Subject: [PATCH 20/26] clarified language around hg19 example in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb58529..af5cc73 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ LOLAweb will look at the value in `$LWREF` for the reference data. This folder s LOLAweb looks for `$LWLOCAL` to have two subfolders: `cache`, and `shinylogs`. This is where the app will write results and log files. If running LOLAweb on a server, be sure these directories are writeable by the Docker process. -The following instructions demonstrate a minimal example of how to download and configure the LOLAweb data directories for a `hg19` reference data: +The following instructions demonstrat how to download and configure the LOLAweb data directories for a minimal example using `hg19` reference data: ``` ## assign env vars for data path From be662bf683d87ceea78769dfe1c4ffcc4d095e71 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:49:23 -0500 Subject: [PATCH 21/26] removed stray colon and standard caps for Docker Swarm in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af5cc73..f2b4190 100644 --- a/README.md +++ b/README.md @@ -120,9 +120,9 @@ http://localhost/LOLAweb/apps/LOLAweb You could also run the `dev` version of the container by pulling `ghcr.io/databio/lolaweb:dev`. This will retrieve the dev tagged image from the GitHub Container Registry. Just add `:dev` to the container name at the end of the `docker run` command above. -### Running multiple LOLAweb containers simultaneously with Docker Swarm: +### Running multiple LOLAweb containers simultaneously with Docker Swarm -For the typical use case of an individual user, a single running container will suffice. But if you need to set up an enterprise-level LOLAweb server that can handle concurrent users, we've also made that easy by using docker swarm. This is how we run the main LOLAweb servers, and you could do the same thing if you want your own local implementation. Docker Swarm is a technique for running multiple instances of the same container. [Read more](swarm/README.md) about how to set up your own swarm. +For the typical use case of an individual user, a single running container will suffice. But if you need to set up an enterprise-level LOLAweb server that can handle concurrent users, we've also made that easy by using Docker Swarm. This is how we run the main LOLAweb servers, and you could do the same thing if you want your own local implementation. Docker Swarm is a technique for running multiple instances of the same container. [Read more](swarm/README.md) about how to set up your own swarm. ### Troubleshooting From e2a8af4d484b3bc3f3b343eb4ff36c65b04638df Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:51:06 -0500 Subject: [PATCH 22/26] typo in shiny README --- apps/LOLAweb/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/LOLAweb/README.md b/apps/LOLAweb/README.md index a430d2a..de303bc 100644 --- a/apps/LOLAweb/README.md +++ b/apps/LOLAweb/README.md @@ -27,6 +27,6 @@ biocLite(c("LOLA", "GenomicRanges")) You'll also need underlying data that is not available in this repository in order to establish the universes, example user sets and reference genome directories. Detailed guidance for how to download and organize reference data is available in the [LOLAweb project README](https://github.com/databio/lolaweb/#readme). -Note that one additional step for running the app locally willbe to either 1) pass the `$LWREF` and `$LWLOCAL` environment variables to your R session or 2) overwite the definition of `localDir` and `refDir` in `misc.R` with the respective paths on your machine. +Note that one additional step for running the app locally will be to either 1) pass the `$LWREF` and `$LWLOCAL` environment variables to your R session or 2) overwite the definition of `localDir` and `refDir` in `misc.R` with the respective paths on your machine. With dependencies installed and data configuration complete you can now launch the app with `shiny::runApp()` from within R at the root of this directory. \ No newline at end of file From 3ac7754042fe99e5b0e916ce8dae01f3ee2fa157 Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Sat, 12 Mar 2022 12:55:43 -0500 Subject: [PATCH 23/26] editing docker build in GH action per repo file structure re-org --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 329e1d9..6864948 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,12 +16,12 @@ jobs: - name: Build DEV if: github.ref == 'refs/heads/dev' run: | - docker build -t ghcr.io/databio/lolaweb:dev docker/dev + docker build -t ghcr.io/databio/lolaweb:dev . - name: Build MASTER if: github.ref == 'refs/heads/master' run: | - docker build -t ghcr.io/databio/lolaweb:latest docker + docker build -t ghcr.io/databio/lolaweb:latest . - name: Login to GHCR uses: docker/login-action@v1 From f677e0aa00eb5016160afebd5d3869719e40f5ba Mon Sep 17 00:00:00 2001 From: VP Nagraj Date: Wed, 23 Mar 2022 22:29:56 -0400 Subject: [PATCH 24/26] editing site_dir in shiny conf to point to app; updated docs --- README.md | 2 +- shiny-server.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f2b4190..c6c5547 100644 --- a/README.md +++ b/README.md @@ -113,7 +113,7 @@ docker run -d \ Open a browser to: ``` -http://localhost/LOLAweb/apps/LOLAweb +http://localhost ``` ### Running a `dev` container diff --git a/shiny-server.conf b/shiny-server.conf index dd02cdf..9bca27f 100644 --- a/shiny-server.conf +++ b/shiny-server.conf @@ -17,7 +17,7 @@ server { # Run this location in 'site_dir' mode, which hosts the entire directory # tree at '/srv/shiny-server' - site_dir /srv/shiny-server; + site_dir /srv/shiny-server/LOLAweb/apps/LOLAweb; # Define where we should put the log files for this location log_dir /var/log/shiny-server; From 10616658189aa271e14ff65b92c47fcac08f581b Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Wed, 13 Apr 2022 14:34:52 -0400 Subject: [PATCH 25/26] Updating README for swarm --- swarm/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swarm/README.md b/swarm/README.md index 9d13d3b..d046b80 100644 --- a/swarm/README.md +++ b/swarm/README.md @@ -20,6 +20,8 @@ The main elements of this stack: - Træfik load balancer - distributes traffic to containers using sticky sessions and websockets 2. Network definition - creates an overlay network for communication between containers +You should update the label `traefik.frontend.rule=Host:` to an appropriate fully qualified domain name before deploying, along with the `docker.domain` in the +`loadbal` stanza. ## Docker Stack Definition @@ -54,7 +56,7 @@ services: - "traefik.port=80" - "traefik.frontend.rule=PathPrefix:/;" - "traefik.backend.loadbalancer.sticky=true" - - "traefik.frontend.rule=Host:lolaweb.databio.org;AddPrefix:/LOLAweb/apps/LOLAweb;" + - "traefik.frontend.rule=Host:lolaweb.databio.org;" loadbal: image: traefik From e34c48065ed1aaa8c07aeefb3650e465254aec46 Mon Sep 17 00:00:00 2001 From: Neal Magee Date: Wed, 13 Apr 2022 16:14:17 -0400 Subject: [PATCH 26/26] Conditional SQS message for branches --- .github/workflows/build.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6864948..d1c4b05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,7 +56,15 @@ jobs: aws-region: us-east-1 - name: Send SQS message + if: github.ref == 'refs/heads/master' + uses: isbang/sqs-action@v0.1.1 + with: + sqs-url: https://queue.amazonaws.com/474683445819/lola-updates + message: 'master' + - + name: Send SQS message + if: github.ref == 'refs/heads/dev' uses: isbang/sqs-action@v0.1.1 with: sqs-url: https://queue.amazonaws.com/474683445819/lola-updates - message: ${GITHUB_REF##*/} + message: 'dev'