From 0c79aecc7cf3679f34d2d245beddece450cb4d16 Mon Sep 17 00:00:00 2001 From: nichlaes <44668005+nichlaes@users.noreply.github.com> Date: Wed, 24 Jan 2024 15:07:15 +0100 Subject: [PATCH] Creates docker containers for client lib and runner (#440) - Creates docker containers for client and lib microservice. - Adds docker compose files for development and localhost installation. - Adds docker image publishing steps to Github Actions. --- .github/workflows/client.yml | 30 +++++ .github/workflows/docker.yml | 46 ++++++++ .github/workflows/lib-ms.yml | 26 +++++ client/.dockerignore | 3 + .../gateway/dynamic/fileConfig.docker.yml | 48 ++++++++ deploy/config/gateway/traefik.yml | 20 ++++ deploy/config/lib.docker | 9 ++ docker/README.md | 106 ++++++++++++++++++ docker/client.dockerfile | 28 +++++ docker/compose.dev.yml | 44 ++++++++ docker/compose.local.yml | 34 ++++++ docker/libms.dockerfile | 14 +++ .../gateway/dynamic/fileConfig.docker.yml | 48 ++++++++ 13 files changed, 456 insertions(+) create mode 100644 .github/workflows/docker.yml create mode 100644 client/.dockerignore create mode 100644 deploy/config/gateway/dynamic/fileConfig.docker.yml create mode 100644 deploy/config/gateway/traefik.yml create mode 100644 deploy/config/lib.docker create mode 100644 docker/README.md create mode 100644 docker/client.dockerfile create mode 100644 docker/compose.dev.yml create mode 100644 docker/compose.local.yml create mode 100644 docker/libms.dockerfile create mode 100644 servers/config/gateway/dynamic/fileConfig.docker.yml diff --git a/.github/workflows/client.yml b/.github/workflows/client.yml index 78f4c5274..a2a6a752f 100644 --- a/.github/workflows/client.yml +++ b/.github/workflows/client.yml @@ -49,3 +49,33 @@ jobs: with: files: client/coverage/clover.xml flags: client-unit-integration-tests + + get_version: + name: Get version + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + - name: Install jq + run: sudo apt-get install -y jq + - name: get version + id: get_version + run: | + version=`cat ./client/package.json | jq -r '.version'` + echo "version=$version" >> $GITHUB_OUTPUT + outputs: + version: ${{ steps.get_version.outputs.version }} + + publish-docker-image: + if: | + github.event_name == 'push' && + (startsWith(github.ref, 'refs/heads/feature/') || startsWith(github.ref, 'refs/heads/release-v')) + + name: Publish Docker image + needs: [client, get_version] + uses: ./.github/workflows/docker.yml + with: + registry: ghcr.io + image-name: into-cps-association/dtaas-client + version: ${{ needs.get_version.outputs.version }} + dockerfile: client.dockerfile \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..8a0f235d8 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,46 @@ +name: Build and Push Docker Image + +on: + workflow_call: + inputs: + registry: + required: false + type: string + default: 'ghcr.io' + image-name: + required: true + type: string + version: + required: true + type: string + default: 'latest' + dockerfile: + required: true + type: string + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./docker/${{ inputs.dockerfile }} + push: true + tags: ${{ inputs.registry }}/${{ inputs.image-name }}:${{ inputs.version }} \ No newline at end of file diff --git a/.github/workflows/lib-ms.yml b/.github/workflows/lib-ms.yml index 1a6fb7057..32c955fab 100644 --- a/.github/workflows/lib-ms.yml +++ b/.github/workflows/lib-ms.yml @@ -92,3 +92,29 @@ jobs: yarn publish --access public env: NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + get_version: + name: Get version + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + - name: Install jq + run: sudo apt-get install -y jq + - name: get version + id: get_version + run: | + version=`cat ./client/package.json | jq -r '.version'` + echo "version=$version" >> $GITHUB_OUTPUT + outputs: + version: ${{ steps.get_version.outputs.version }} + + publish-docker-image: + name: Publish Docker image + needs: [ publish-package, get_version] + uses: ./.github/workflows/docker.yml + with: + registry: ghcr.io + image-name: into-cps-association/dtaas-libms + version: ${{ needs.get_version.outputs.version }} + dockerfile: libms.dockerfile \ No newline at end of file diff --git a/client/.dockerignore b/client/.dockerignore new file mode 100644 index 000000000..f4cc0a5fd --- /dev/null +++ b/client/.dockerignore @@ -0,0 +1,3 @@ +*/node_modules +*/build +*/test \ No newline at end of file diff --git a/deploy/config/gateway/dynamic/fileConfig.docker.yml b/deploy/config/gateway/dynamic/fileConfig.docker.yml new file mode 100644 index 000000000..359314a0b --- /dev/null +++ b/deploy/config/gateway/dynamic/fileConfig.docker.yml @@ -0,0 +1,48 @@ +http: + routers: + dtaas: + entryPoints: + - http + rule: 'Host(`localhost`)' + middlewares: + - basic-auth + service: dtaas + + user1: + entryPoints: + - http + rule: 'Host(`localhost`) && PathPrefix(`/user1`)' + middlewares: + - basic-auth + service: user1 + + libms: + entryPoints: + - http + rule: 'Host(`localhost`) && PathPrefix(`/lib`)' + service: libms + + + # Middleware: Basic authentication + middlewares: + basic-auth: + basicAuth: + usersFile: "/etc/traefik/auth" + removeHeader: true + + + services: + dtaas: + loadBalancer: + servers: + - url: "http://client:4000" + + user1: + loadBalancer: + servers: + - url: "http://ml-workspace-user1:8080" + + libms: + loadBalancer: + servers: + - url: "http://libms:4001" \ No newline at end of file diff --git a/deploy/config/gateway/traefik.yml b/deploy/config/gateway/traefik.yml new file mode 100644 index 000000000..4f79c1d6f --- /dev/null +++ b/deploy/config/gateway/traefik.yml @@ -0,0 +1,20 @@ +entryPoints: + http: + address: :80 + +providers: + providersThrottleDuration: 2s + + # File provider for connecting things that are outside of docker / defining middleware + file: + filename: /etc/traefik/dynamic/fileConfig.yml + watch: true + +# Enable traefik ui +#dapi: +# dashboard: true +# insecure: true + +# Log level INFO|DEBUG|ERROR +log: + level: DEBUG diff --git a/deploy/config/lib.docker b/deploy/config/lib.docker new file mode 100644 index 000000000..3f86c1693 --- /dev/null +++ b/deploy/config/lib.docker @@ -0,0 +1,9 @@ +PORT='4001' +MODE='local' +LOCAL_PATH ='/dtaas/libms/files' +GITLAB_GROUP ='dtaas' +GITLAB_URL='https://gitlab.com/api/graphql' +TOKEN='123-sample-token' +LOG_LEVEL='debug' +APOLLO_PATH='/lib' +GRAPHQL_PLAYGROUND='true' \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..a9ebdc347 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,106 @@ +# Docker workflow for DTaaS + +This readme will explain the building and use of different docker files +for use in development and installation of the DTaaS software. + +**NOTE**: A local docker and docker-compose installation is a pre-requisite +for using docker workflows. + +## Folder Structure + +There are two dockerfiles for building the containers: + +- **client.dockerfile**: Dockerfile for building + the client application container. + +- **libms.dockerfile**: Dockerfile for building the library microservice container. + +There are also two compose files for development and local installation scenarios. + +- **compose.dev.yml:** Docker Compose configuration for development environment. + +- **compose.local.yml:** Docker Compose configuration for localhost installation. + +## Build and Publish Docker Images + +### Users + +Build and publish the docker images. This step is required only for +the publication of images to Docker Hub. This publishing step is managed +only by project maintainers. Regular users can skip this step. + +```sh +docker login -u -p +docker build -t intocps/libms:latest -f ./docker/libms.dockerfile . +docker tag intocps/libms:latest intocps/libms:version +docker push intocps/libms:latest +docker push intocps/libms:version + +docker build -t intocps/dtaas-web:latest -f ./docker/client.dockerfile . +docker tag intocps/dtaas-web:latest intocps/dtaas-web:version +docker push intocps/dtaas-web:latest +docker push intocps/dtaas-web:version +``` + +To tag version 0.3.1 for example, use + +```sh +docker tag intocps/dtaas-web:latest intocps/dtaas-web:0.3.1 +``` + +### Developers + +Use of docker images is handy for developers as well. It is suggested +that developers build the required images locally on their computer and +use them for development purposes. The images can be built using + +```sh +docker-compose -f compose.dev.yml build +``` + +## Running Docker Containers + +Follow these steps to use the application with docker. + +The DTaaS application requires multiple configuration files. The list of +configuration files to be modified are given for each scenario. + +### Development Environment + +This scenario is for software developers: + +The configuration files to be updated are: + +1. client/config/dev.js +1. deploy/config/lib.docker +1. servers/config/gateway/auth + +The relevant docker commands are: + +```bash +docker-compose -f compose.dev.yml up -d #start the application +docker-compose -f compose.dev.yml down #terminate the application +``` + +### Localhost Use + +This scenario is for users interested in using the software on +their computers (localhost): + +The configuration files to be updated are: + +1. deploy/config/client/env.local.js +1. deploy/config/lib.docker +1. deploy/config/gateway/auth + +The relevant docker commands are: + +```bash +docker-compose -f compose.local.yml up -d #start the application +docker-compose -f compose.local.yml down #terminate the application +``` + +### Access the Application + +You should access the application through the PORT mapped to the Traefik container. +e.g. `localhost:9000` diff --git a/docker/client.dockerfile b/docker/client.dockerfile new file mode 100644 index 000000000..6a5c37f7d --- /dev/null +++ b/docker/client.dockerfile @@ -0,0 +1,28 @@ +#! docker should be run from the root directory of the project +FROM node:20.10.0-slim as build + +# Set the working directory inside the container +WORKDIR /dtaas/client + +# Copy package.json and package-lock.json to the working directory +COPY ./client/package.json ./ + +# Install dependencies +RUN yarn install --immutable --immutable-cache --check-cache + +# Copy the rest of the application code to the working directory +COPY ./client/ . + +# Build the React app +RUN yarn build + + +FROM node:20.10.0-slim +# Copy the build output to serve +COPY --from=build /dtaas/client/build /dtaas/client/build +COPY --from=build /dtaas/client/package.json /dtaas/client/package.json + +WORKDIR /dtaas/client +RUN npm i -g serve +# Define the command to run your app +CMD ["yarn", "start"] \ No newline at end of file diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml new file mode 100644 index 000000000..8751ecf06 --- /dev/null +++ b/docker/compose.dev.yml @@ -0,0 +1,44 @@ +version: '3' + +services: + client: + build: + context: ../ + dockerfile: ./docker/client.dockerfile + ports: + - "4000:4000" + volumes: + - "../client/config/dev.js:/dtaas/client/build/env.js" + libms: + build: + context: ../ + dockerfile: ./docker/libms.dockerfile + ports: + - "4001:4001" + volumes: + - "../deploy/config/lib.docker:/dtaas/libms/.env" + - "../files:/dtaas/libms/files" + + ml-workspace-user1: + image: mltooling/ml-workspace-minimal:0.13.2 + container_name: ml-workspace-user1 + ports: + - "8090:8080" + volumes: + - "../files/user1:/workspace" + - "../files/common:/workspace/common" + environment: + - AUTHENTICATE_VIA_JUPYTER + - WORKSPACE_BASE_URL=user1 + shm_size: 512m + + traefik-gateway: + image: traefik:v2.10 + container_name: traefik-gateway + ports: + - "9000:80" + volumes: + - "../servers/config/gateway/traefik.yml:/etc/traefik/traefik.yml" + - "../servers/config/gateway/auth:/etc/traefik/auth" + - "../servers/config/gateway/dynamic/fileConfig.docker.yml:/etc/traefik/dynamic/fileConfig.yml" + - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file diff --git a/docker/compose.local.yml b/docker/compose.local.yml new file mode 100644 index 000000000..bceccafbc --- /dev/null +++ b/docker/compose.local.yml @@ -0,0 +1,34 @@ +version: '3' + +services: + client: + image: intocps/dtaas-web + volumes: + - "../deploy/config/client/env.local.js:/dtaas/client/build/env.js" + libms: + image: intocps/libms + volumes: + - "../deploy/config/lib.docker:/dtaas/libms/.env" + - "../files:/dtaas/libms/files" + + ml-workspace-user1: + image: mltooling/ml-workspace-minimal:0.13.2 + container_name: ml-workspace-user1 + volumes: + - "../files/user1:/workspace" + - "../files/common:/workspace/common" + environment: + - AUTHENTICATE_VIA_JUPYTER + - WORKSPACE_BASE_URL=user1 + shm_size: 512m + + traefik-gateway: + image: traefik:v2.10 + container_name: traefik-gateway + ports: + - "9000:80" + volumes: + - "../deploy/config/gateway/traefik.yml:/etc/traefik/traefik.yml" + - "../deploy/config/gateway/auth:/etc/traefik/auth" + - "../deploy/config/gateway/dynamic/fileConfig.docker.yml:/etc/traefik/dynamic/fileConfig.yml" + - "/var/run/docker.sock:/var/run/docker.sock" \ No newline at end of file diff --git a/docker/libms.dockerfile b/docker/libms.dockerfile new file mode 100644 index 000000000..45e5ab5d2 --- /dev/null +++ b/docker/libms.dockerfile @@ -0,0 +1,14 @@ +FROM node:20.10.0-slim + +#! docker should be run from the root directory of the project + +# Set the working directory inside the container +WORKDIR /dtaas/libms + +# pull the libms package from npm registry +RUN npm i -g @into-cps-association/libms@0.3.1 + +COPY ./deploy/config/lib . + +# Define the command to run your app +CMD ["libms"] diff --git a/servers/config/gateway/dynamic/fileConfig.docker.yml b/servers/config/gateway/dynamic/fileConfig.docker.yml new file mode 100644 index 000000000..359314a0b --- /dev/null +++ b/servers/config/gateway/dynamic/fileConfig.docker.yml @@ -0,0 +1,48 @@ +http: + routers: + dtaas: + entryPoints: + - http + rule: 'Host(`localhost`)' + middlewares: + - basic-auth + service: dtaas + + user1: + entryPoints: + - http + rule: 'Host(`localhost`) && PathPrefix(`/user1`)' + middlewares: + - basic-auth + service: user1 + + libms: + entryPoints: + - http + rule: 'Host(`localhost`) && PathPrefix(`/lib`)' + service: libms + + + # Middleware: Basic authentication + middlewares: + basic-auth: + basicAuth: + usersFile: "/etc/traefik/auth" + removeHeader: true + + + services: + dtaas: + loadBalancer: + servers: + - url: "http://client:4000" + + user1: + loadBalancer: + servers: + - url: "http://ml-workspace-user1:8080" + + libms: + loadBalancer: + servers: + - url: "http://libms:4001" \ No newline at end of file