From 10b49241425b98c2f9d1d5db6093dc2918a3012a Mon Sep 17 00:00:00 2001 From: Prasad Talasila Date: Mon, 9 Oct 2023 13:33:53 +0200 Subject: [PATCH] Updates install scripts and docs (#150) - Updates vagrant install scripts. Separates developer and user installation scripts - Updates the developer environment creation script in script/env.sh - Corrects docs formatting issues and OAuth documentation --- .gitignore | 5 +- README.md | 10 +- client/script/clean.bash | 2 +- deploy/README.md | 34 ++- deploy/config/client/env.js | 2 +- deploy/install.sh | 12 +- deploy/single-script-install.sh | 49 ++-- deploy/vagrant/make_boxes/dtaas/README.md | 73 ++++-- deploy/vagrant/make_boxes/dtaas/Vagrantfile | 3 +- deploy/vagrant/make_boxes/dtaas/developer.sh | 45 ++++ deploy/vagrant/make_boxes/dtaas/provision.sh | 73 ------ deploy/vagrant/make_boxes/dtaas/user.sh | 80 ++++++ docs/FAQ.md | 256 ++++++++++--------- docs/PUBLISH.md | 12 + docs/admin/client/CLIENT.md | 2 +- docs/admin/client/auth.md | 49 ++-- docs/admin/host.md | 36 ++- docs/admin/services.md | 22 +- docs/admin/trial.md | 36 ++- docs/admin/vagrant/base-box.md | 77 ++++-- docs/developer/system/current-status.md | 2 +- docs/index.md | 6 +- docs/thanks.md | 7 +- docs/user/digital-twins/create.md | 10 +- docs/user/examples/index.md | 3 +- docs/user/servers/lib/LIB-MS.md | 112 ++++---- script/docker.sh | 12 +- script/env.sh | 42 ++- servers/config/gateway/README.md | 2 +- servers/execution/runner/script/clean.bash | 2 +- servers/lib/script/clean.bash | 2 +- servers/lib/test/starttraefik.bash | 2 +- 32 files changed, 683 insertions(+), 397 deletions(-) create mode 100755 deploy/vagrant/make_boxes/dtaas/developer.sh delete mode 100755 deploy/vagrant/make_boxes/dtaas/provision.sh create mode 100755 deploy/vagrant/make_boxes/dtaas/user.sh diff --git a/.gitignore b/.gitignore index 80faf2181..69f3a98e1 100644 --- a/.gitignore +++ b/.gitignore @@ -148,4 +148,7 @@ site/ !.vscode/settings.json !.vscode/tasks.json !.vscode/launch.json -!.vscode/extensions.json \ No newline at end of file +!.vscode/extensions.json + +# yarn cache +.yarn \ No newline at end of file diff --git a/README.md b/README.md index 2a75f20c7..6bb1efd2a 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,10 @@ if there is a suggestion to improve the software. This is a mono repo containing code for both the web client and the microservices code base. -Only the [web client](client) and -[library microservice](servers/lib) -components are functional at present. +The [web client](client), +[library](servers/lib) and +[runner](servers/execution/runner) +microservices are functional at present. Everything else is a work-in-progress. Please see the @@ -46,7 +47,8 @@ The DTaaS software platform uses [Træfik](https://github.com/traefik/traefik), [ML Workspace](https://github.com/ml-tooling/ml-workspace), [Grafana](https://github.com/grafana/grafana), -[InfluxDB](https://github.com/influxdata/influxdb) and +[InfluxDB](https://github.com/influxdata/influxdb), +[MQTT](https://github.com/eclipse/mosquitto) and [RabbitMQ](https://github.com/rabbitmq/rabbitmq-server) open-source components. These software components have their own licenses. diff --git a/client/script/clean.bash b/client/script/clean.bash index dd1cdb9b2..5bcab1efd 100755 --- a/client/script/clean.bash +++ b/client/script/clean.bash @@ -1,2 +1,2 @@ #!/bin/bash -rm -rf build/ node_modules/ coverage/ playwright-report/ *.svg \ No newline at end of file +rm -rf build/ node_modules/ coverage/ playwright-report/ "*.svg" \ No newline at end of file diff --git a/deploy/README.md b/deploy/README.md index 2af369e2d..0a688d8f2 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,7 +1,7 @@ # DTaaS on Linux Operating System This directory contains code for running DTaaS application -on a Ubuntu Server 20.04 Operating System. +on a Ubuntu Server 22.04 Operating System. The setup requires a machine which can spare 16GB RAM, 8 vCPUs and 50GB Hard Disk space. @@ -31,7 +31,8 @@ The installation guide assumes that you can run the application in HTTPS mode. The Traefik gateway configuration is at [fileConfig](../config/gateway/fileConfig.yml). Change `localhost` to `foo.com` and user1/user2 to the usernames chosen by you. -**NOTE**: Do not use `http://` or `https://` in [fileConfig](../config/gateway/fileConfig.yml). +**NOTE**: Do not use `http://` or `https://` +in [fileConfig](../config/gateway/fileConfig.yml). #### Authentication @@ -73,13 +74,38 @@ APOLLO_PATH='/lib' GRAPHQL_PLAYGROUND='true' ``` -## Configure react website +## Configure React Client Website + +### Gitlab OAuth application + +The DTaaS react website requires Gitlab OAuth provider. +If you need more help with this step, please see +the [Authentication page](../docs/admin/client/auth.md). + +You need the following information from the OAuth application +registered on Gitlab: + +| Gitlab Variable Name | Variable name in Client env.js | Default Value | +|:---|:---|:---| +| OAuth Provider | REACT_APP_AUTH_AUTHORITY | https://gitlab.foo.com/ | +| Application ID | REACT_APP_CLIENT_ID | +| Callback URL | REACT_APP_REDIRECT_URI | https://foo.com/Library | +| Scopes | REACT_APP_GITLAB_SCOPES | openid, profile, read_user, read_repository, api | + +You can also see +[Gitlab help page](https://docs.gitlab.com/ee/integration/oauth_provider.html) +for getting the Gitlab OAuth application details. + +If you need more help with this step, please see +the [Authentication page](client/auth.md). + +### Update Client Config Change the React website configuration in _deploy/config/client/env.js_. ```js window.env = { - REACT_APP_ENVIRONMENT: 'prod', + REACT_APP_ENVIRONMENT: 'dev', REACT_APP_URL: 'https://foo.com/', REACT_APP_URL_BASENAME: 'dtaas', REACT_APP_URL_DTLINK: '/lab', diff --git a/deploy/config/client/env.js b/deploy/config/client/env.js index dc97190eb..baa19d645 100644 --- a/deploy/config/client/env.js +++ b/deploy/config/client/env.js @@ -1,7 +1,7 @@ window.env = { REACT_APP_ENVIRONMENT: 'dev', REACT_APP_URL: 'https://foo.com/', - REACT_APP_URL_BASENAME: 'dtaas', + REACT_APP_URL_BASENAME: '', REACT_APP_URL_DTLINK: '/lab', REACT_APP_URL_LIBLINK: '', REACT_APP_WORKBENCHLINK_TERMINAL: '/terminals/main', diff --git a/deploy/install.sh b/deploy/install.sh index ab15df110..7767692f1 100755 --- a/deploy/install.sh +++ b/deploy/install.sh @@ -14,11 +14,13 @@ sleep 60 printf "\n \n Install the system dependencies...\n " printf "....\n " -bash script/env.sh || exit +bash deploy/make_boxes/dtaas/user.sh || exit printf "\n \n Download the required docker images...\n " printf ".........\n " -source script/docker.sh || exit +docker pull traefik:v2.10 +docker pull mltooling/ml-workspace:0.13.2 +printf "\n\n docker images successfully downloaded...\n \n \n " printf "\n \n Continue with the DTaaS installation...\n " @@ -52,7 +54,7 @@ docker run -d \ -p 8090:8080 \ --name "ml-workspace-user1" \ -v "${TOP_DIR}/files/user1:/workspace" \ - -v "${TOP_DIR}/files/common:/workspace/common:ro" \ + -v "${TOP_DIR}/files/common:/workspace/common" \ --env AUTHENTICATE_VIA_JUPYTER="" \ --env WORKSPACE_BASE_URL="user1" \ --shm-size 512m \ @@ -63,7 +65,7 @@ docker run -d \ -p 8091:8080 \ --name "ml-workspace-user2" \ -v "${TOP_DIR}/files/user2:/workspace" \ - -v "${TOP_DIR}/files/common:/workspace/common:ro" \ + -v "${TOP_DIR}/files/common:/workspace/common" \ --env AUTHENTICATE_VIA_JUPYTER="" \ --env WORKSPACE_BASE_URL="user2" \ --shm-size 512m \ @@ -83,7 +85,7 @@ sudo docker run -d \ -v "$PWD/auth:/etc/traefik/auth" \ -v "$PWD/dynamic:/etc/traefik/dynamic" \ -v /var/run/docker.sock:/var/run/docker.sock \ - traefik:v2.5 || true + traefik:v2.10 || true #---------- printf "\n \n Create crontabs to run the application in daemon mode.\n " diff --git a/deploy/single-script-install.sh b/deploy/single-script-install.sh index 07743c1fb..6cf781007 100755 --- a/deploy/single-script-install.sh +++ b/deploy/single-script-install.sh @@ -26,9 +26,11 @@ sudo mkdir -p /etc/apt/keyrings if [ ! -f /etc/apt/keyrings/docker.gpg ] then curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - sudo printf \ - "deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - %s stable" "$(dpkg --print-architecture)" "$(lsb_release -cs)" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + printf \ + "deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu %s stable" \ + "$(dpkg --print-architecture)" "$(lsb_release -cs)" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null fi sudo apt-get update -y @@ -58,14 +60,30 @@ sudo systemctl enable docker.service sudo systemctl enable containerd.service -# Install nodejs environment -curl -fsSL https://deb.nodesource.com/setup_18.x | sudo bash - +sudo apt-get update +sudo apt-get install -y ca-certificates curl gnupg +sudo mkdir -p /etc/apt/keyrings +if [ ! -f /etc/apt/keyrings/nodesource.gpg ] +then + curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ + sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +fi +NODE_MAJOR=18 +printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ + https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ + sudo tee /etc/apt/sources.list.d/nodesource.list +sudo apt-get update sudo apt-get install -y nodejs +sudo npm install -g npm@10.2.0 + if [ ! -f /usr/share/keyrings/yarnkey.gpg ] then - curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null - printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main" | sudo tee /etc/apt/sources.list.d/yarn.list + curl -sL "https://dl.yarnpkg.com/debian/pubkey.gpg" | gpg --dearmor | \ + sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null + printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] \ + https://dl.yarnpkg.com/debian stable main \n" | \ + sudo tee /etc/apt/sources.list.d/yarn.list fi sudo apt-get update -y @@ -78,12 +96,8 @@ printf "\n\n End of installing dependencies...\n\n\n " # get the required docker images printf "Download the required docker images...\n " printf ".........\n\n\n " -docker pull traefik:v2.5 -docker pull influxdb:2.4 +docker pull traefik:v2.10 docker pull mltooling/ml-workspace:0.13.2 -docker pull grafana/grafana -docker pull telegraf -docker pull gitlab/gitlab-ce:15.10.0-ce.0 printf "\n\n docker images successfully downloaded...\n \n \n " #---- @@ -128,7 +142,7 @@ yarn build { printf "PORT='4001'\n " printf "MODE='local'\n " - printf "LOCAL_PATH ='%s'\n " "$TOP_DIR" + printf "LOCAL_PATH ='%s/files'\n " "$TOP_DIR" printf "GITLAB_GROUP ='dtaas'\n " printf "GITLAB_URL='https://gitlab.com/api/graphql'\n " printf "TOKEN='123-sample-token'\n " @@ -146,7 +160,7 @@ docker run -d \ -p 8090:8080 \ --name "ml-workspace-user1" \ -v "${TOP_DIR}/files/user1:/workspace" \ - -v "${TOP_DIR}/files/common:/workspace/common:ro" \ + -v "${TOP_DIR}/files/common:/workspace/common" \ --env AUTHENTICATE_VIA_JUPYTER="" \ --env WORKSPACE_BASE_URL="user1" \ --shm-size 512m \ @@ -157,7 +171,7 @@ docker run -d \ -p 8091:8080 \ --name "ml-workspace-user2" \ -v "${TOP_DIR}/files/user2:/workspace" \ - -v "${TOP_DIR}/files/common:/workspace/common:ro" \ + -v "${TOP_DIR}/files/common:/workspace/common" \ --env AUTHENTICATE_VIA_JUPYTER="" \ --env WORKSPACE_BASE_URL="user2" \ --shm-size 512m \ @@ -177,7 +191,7 @@ docker run -d \ -v "$PWD/auth:/etc/traefik/auth" \ -v "$PWD/dynamic:/etc/traefik/dynamic" \ -v /var/run/docker.sock:/var/run/docker.sock \ - traefik:v2.5 || true + traefik:v2.10 || true #---------- @@ -191,6 +205,7 @@ printf "\n\n The installation is complete.\n\n\n " printf "Continue with the application configuration.\n " printf ".........\n\n\n " -printf "Remember to change foo.com to your local hostname in the following files.\n " +printf "Remember to change foo.com and Gitlab OAuth details to your \ +local settings in the following files.\n " printf "1. %s/client/build/env.js\n " "$TOP_DIR" printf "2. %s/servers/config/gateway/dynamic/fileConfig.yml\n " "$TOP_DIR" diff --git a/deploy/vagrant/make_boxes/dtaas/README.md b/deploy/vagrant/make_boxes/dtaas/README.md index 713ad986f..954e3719f 100644 --- a/deploy/vagrant/make_boxes/dtaas/README.md +++ b/deploy/vagrant/make_boxes/dtaas/README.md @@ -1,21 +1,63 @@ -# DTaaS Boxes +# DTaaS Vagrant Box -This is a box that has the following items: +This README provides instructions on creating a custom Operating System +virtual disk for running the DTaaS software. The virtual disk is managed +by **vagrant**. The purpose is two fold: -* docker -* nodejs and yarn -* jupyter -* microk8s +* Provide cross-platform installation of the DTaaS application. + Any operating system supporting use of vagrant software utility can + support installation of the DTaaS software. +* Create a ready to use development environment for code contributors. + +There are two scripts in this directory: + +| Script name | Purpose | Default | +|:---|:---|:---| +| `user.sh` | user installation | :white_check_mark: | +| `developer.sh` | developer installation | :x: | + +If you are installing the DTaaS for developers, the default installation +caters to your needs. You can skip the next step and continue with the +creation of vagrant box. + +If you are a developer and would like additional software installed, you need +to modify `Vagrantfile`. The existing `Vagrantfile` has two lines: + +```sh + config.vm.provision "shell", path: "user.sh" + #config.vm.provision "shell", path: "developer.sh" +``` + +Uncomment the second line to have more software components installed. If you +are not a developer, no changes are required to the `Vagrantfile`. + +This vagrant box installed for users will have the following items: + +* docker v24.0 +* nodejs v18.8 +* yarn v1.22 +* npm v10.2 * containers - * mltooling/ml-workspace:0.13.2 - * traefik2.5 - * influxdb2.4 - * grafana - * telegraf - * gitlab + * ml-workspace v0.13 + * traefik v2.10 + * gitlab-ce v16.4 + * influxdb v2.7 + * grafana v10.1 + * rabbitmq v3-management + * eclipse-mosquitto (mqtt) v2 + +This vagrant box installed for developers will have +the following items additional items: + +* docker-compose v2.20 +* microk8s v1.27 +* jupyterlab +* mkdocs +* containers + * telegraf v1.28 -Publish a base virtualbox package to be used by -vagrant to publish all other virtualbox packages +The upcoming instructions will help with the creation of +base vagrant box. ```bash #create a key pair @@ -26,7 +68,6 @@ mv key.pub vagrant.pub vagrant up # let the provisioning be complete -# replace the vagrant ssh key-pair with personal one vagrant ssh # install the oh-my-zsh @@ -37,6 +78,7 @@ git clone https://github.com/zsh-users/zsh-autosuggestions ${ZSH_CUSTOM:-~/.oh-m # inside ~/.zshrc, modify the following line plugins=(git zsh-autosuggestions history cp tmux) +# to replace the vagrant ssh key-pair with personal one # remove the vagrant default public key - first line of # /home/vagrant/.ssh/authorized_keys @@ -47,6 +89,7 @@ cp vagrant .vagrant/machines/default/virtualbox/private_key # check vagrant ssh #should work +# exit vagrant guest machine and then vagrant halt vagrant package --base dtaas \ diff --git a/deploy/vagrant/make_boxes/dtaas/Vagrantfile b/deploy/vagrant/make_boxes/dtaas/Vagrantfile index 2767c042c..e37807bb5 100644 --- a/deploy/vagrant/make_boxes/dtaas/Vagrantfile +++ b/deploy/vagrant/make_boxes/dtaas/Vagrantfile @@ -12,6 +12,7 @@ Vagrant.configure("2") do |config| vb.cpus = 8 end - config.vm.provision "shell", path: "provision.sh" + config.vm.provision "shell", path: "user.sh" + #config.vm.provision "shell", path: "developer.sh" end diff --git a/deploy/vagrant/make_boxes/dtaas/developer.sh b/deploy/vagrant/make_boxes/dtaas/developer.sh new file mode 100755 index 000000000..70d387dac --- /dev/null +++ b/deploy/vagrant/make_boxes/dtaas/developer.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Installs necessary packages to create the docker environment for +# executing the DTaaS application + +apt-get update -y +apt-get upgrade -y + +#install docker-compose from https://docs.docker.com/compose/install/other/ +curl -SL "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64" \ + -o /usr/local/bin/docker-compose +ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose +chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose + +# Install openssl for certificate generation +apt-get install -y wget openssl + +# Install playwright tool for integration tests on browsers +npx --yes playwright install-deps + +#------------- +printf "\n\n Install jupyterlab and mkdocs" +pip install jupyterlab +pip install mkdocs +pip3 install mkdocs-material +pip3 install python-markdown-math +pip3 install mkdocs-open-in-new-tab +pip3 install mkdocs-with-pdf +pip3 install qrcode + +# Install minimal Kubernetes cluster +snap install microk8s --classic +usermod -a -G microk8s vagrant +chown -f -R vagrant ~/.kube +newgrp microk8s + +# get the required docker images +docker pull telegraf:1.28.2 + +# Install markdownlint +sudo apt-get install -y rubygems +sudo gem install mdl + +# Install madge for generating dependency graphs of typescript projects +sudo apt-get install -y graphviz +sudo npm install -g madge \ No newline at end of file diff --git a/deploy/vagrant/make_boxes/dtaas/provision.sh b/deploy/vagrant/make_boxes/dtaas/provision.sh deleted file mode 100755 index 9243314ef..000000000 --- a/deploy/vagrant/make_boxes/dtaas/provision.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# Installs necessary packages to create the docker environment for -# executing the DTaaS application - -apt-get update -y -apt-get upgrade -y - -# https://docs.docker.com/engine/install/ubuntu/ -apt-get install -y \ - ca-certificates \ - curl \ - gnupg \ - lsb-release \ - zsh \ - apache2-utils - -mkdir -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -printf \ - "deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - %s stable \n" "$(dpkg --print-architecture)" "$(lsb_release -cs)" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -apt-get update -y -apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin -groupadd docker -usermod -aG docker vagrant -newgrp docker -service docker start -docker run hello-world - -systemctl enable docker.service -systemctl enable containerd.service - -#install docker-compose from https://docs.docker.com/compose/install/other/ -sudo curl -SL "https://github.com/docker/compose/releases/download/v2.15.1/docker-compose-linux-x86_64" -o /usr/local/bin/docker-compose -sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose - -curl -fsSL https://deb.nodesource.com/setup_18.x | bash - -apt-get install -y nodejs -curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null -printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main \n" | sudo tee /etc/apt/sources.list.d/yarn.list -apt-get update -y -apt-get install -y yarn -npm install -g serve - -#------------- -printf "\n\n Install jupyter toolchain" -sudo apt-get install -y python3-pip -sudo -H pip install jupyterlab -sudo -H pip install mkdocs -sudo -H pip3 install mkdocs-material -sudo -H pip3 install python-markdown-math -sudo -H pip3 install mkdocs-open-in-new-tab -sudo -H pip3 install mkdocs-with-pdf -sudo -H pip3 install qrcode - -# Install minimal Kubernetes cluster -snap install microk8s --classic -usermod -a -G microk8s vagrant -chown -f -R vagrant ~/.kube -newgrp microk8s - -cat /vagrant/vagrant.pub >> /home/vagrant/.ssh/authorized_keys -mkdir -p /root/.ssh -cat /vagrant/vagrant.pub >> /root/.ssh/authorized_keys - -# get the required docker images -docker pull traefik:v2.5 -docker pull influxdb:2.4 -docker pull mltooling/ml-workspace:0.13.2 -docker pull grafana/grafana -docker pull telegraf -docker pull gitlab/gitlab-ce:15.10.0-ce.0 - diff --git a/deploy/vagrant/make_boxes/dtaas/user.sh b/deploy/vagrant/make_boxes/dtaas/user.sh new file mode 100755 index 000000000..62a880aed --- /dev/null +++ b/deploy/vagrant/make_boxes/dtaas/user.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Installs necessary packages to create the docker environment for +# executing the DTaaS application + +apt-get update -y +apt-get upgrade -y + +# https://docs.docker.com/engine/install/ubuntu/ +apt-get install -y \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + zsh \ + apache2-utils \ + net-tools \ + python3-dev \ + python3-pip + +mkdir -p /etc/apt/keyrings +if [ ! -f /etc/apt/keyrings/docker.gpg ] +then + curl -fsSL "https://download.docker.com/linux/ubuntu/gpg" | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + printf \ + "deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu %s stable" \ + "$(dpkg --print-architecture)" "$(lsb_release -cs)" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null +fi + +apt-get update -y +apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin +groupadd docker || true +usermod -aG docker vagrant || true +newgrp docker || true +service docker start +docker run hello-world + +systemctl enable docker.service +systemctl enable containerd.service + +apt-get update +apt-get install -y ca-certificates curl gnupg +mkdir -p /etc/apt/keyrings +if [ ! -f /etc/apt/keyrings/nodesource.gpg ] +then + curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ + gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +fi +NODE_MAJOR=18 +printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ + https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ + tee /etc/apt/sources.list.d/nodesource.list +apt-get update +apt-get install -y nodejs +npm install -g npm@10.2.0 + +if [ ! -f /usr/share/keyrings/yarnkey.gpg ] +then + curl -sL "https://dl.yarnpkg.com/debian/pubkey.gpg" | gpg --dearmor | \ + tee /usr/share/keyrings/yarnkey.gpg >/dev/null + printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main \n" | \ + tee /etc/apt/sources.list.d/yarn.list +fi +apt-get update -y +apt-get install -y yarn +npm install -g serve + +cat /vagrant/vagrant.pub >> /home/vagrant/.ssh/authorized_keys +mkdir -p /root/.ssh +cat /vagrant/vagrant.pub >> /root/.ssh/authorized_keys + +# get the required docker images +docker pull traefik:v2.10 +docker pull mltooling/ml-workspace:0.13.2 +docker pull grafana/grafana:10.1.4 +docker pull influxdb:2.7 +docker pull rabbitmq:3-management +docker pull eclipse-mosquitto:2 +docker pull gitlab/gitlab-ce:16.4.1-ce.0 diff --git a/docs/FAQ.md b/docs/FAQ.md index 98fee4a0e..305926606 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -10,196 +10,204 @@ ## General Questions + + ??? Question "What is DTaaS?" -DTaaS is software platform on which you can create and run digital twins. -Please see the [features](user/features.md) page -to get a sense of the things you can do in DaaS. + DTaaS is software platform on which you can create and run digital twins. + Please see the [features](user/features.md) page + to get a sense of the things you can do in DaaS. ??? Question "Are there any Key Performance / Capability Indicators for DTaaS?" -| Key Performance Indicator | Value | -|:---|:---| -| Processor | Two AMD EPYC 7443 24-Core Processors | -| Maximum Storage Capacity | 4TB SSD, RAID 0 configuration | -| Storage Type | File System | -| Maximum file size | 10 GB | -| Data transfer speed | 100 Mbps | -| Data Security | Yes | -| Data Privacy | Yes | -| Redundancy | None | -| Availability | It is a matter of human resources. If you have human -resources to maintain DTaaS round the clock, -upwards 95% is easily possible. | + | Key Performance Indicator | Value | + |:---|:---| + | Processor | Two AMD EPYC 7443 24-Core Processors | + | Maximum Storage Capacity | 4TB SSD, RAID 0 configuration | + | Storage Type | File System | + | Maximum file size | 10 GB | + | Data transfer speed | 100 Mbps | + | Data Security | Yes | + | Data Privacy | Yes | + | Redundancy | None | + | Availability | It is a matter of human resources. If you have human + resources to maintain DTaaS round the clock, + upwards 95% is easily possible. | ??? Question "Do you provide licensed software like Matlab?" -The licensed software are not available on the software platform. -But users have private workspaces which are based -on Linux-based xfce Desktop environment. -Users can install software in their workspaces. The licensed software -installed by one user is not available to another user. + The licensed software are not available on the software platform. + But users have private workspaces which are based + on Linux-based xfce Desktop environment. + Users can install software in their workspaces. The licensed software + installed by one user is not available to another user. ## Digital Twin Models ??? Question "Can DTaaS create new DT models?" -DTaaS is not a model creation tool. You can put model creation tool -inside DTaaS and create new models. -The DTaaS itself does not create digital twin models but it can help -users create digital twin models. You can run -Linux desktop / terminal tools inside the DTaaS. So you can create -models inside DTaaS and run them using tools that can run in Linux. -The Windows only tools can not run in DTaaS. + DTaaS is not a model creation tool. You can put model creation tool + inside DTaaS and create new models. + The DTaaS itself does not create digital twin models but it can help + users create digital twin models. You can run + Linux desktop / terminal tools inside the DTaaS. So you can create + models inside DTaaS and run them using tools that can run in Linux. + The Windows only tools can not run in DTaaS. ??? Question "How can DTaaS help to design geometric model? Does it support 3D modeling and simulation?" -Well, DTaaS by itself does not produce any models. DTaaS only provides -a platform and an ecosystem of services to facilitate digital twins to -be run as services. Since each user has a Linux OS at their disposal, -they can also run digital twins that have graphical interface. -In summary, DTaaS is neither a modeling nor simulation tool. If you need -these kinds of tools, you need to bring them onto the platform. -For example, if you need Matlab for your work, you need to bring -he licensed Matlab software. + Well, DTaaS by itself does not produce any models. DTaaS only provides + a platform and an ecosystem of services to facilitate digital twins to + be run as services. Since each user has a Linux OS at their disposal, + they can also run digital twins that have graphical interface. + In summary, DTaaS is neither a modeling nor simulation tool. If you need + these kinds of tools, you need to bring them onto the platform. + For example, if you need Matlab for your work, you need to bring + he licensed Matlab software. ??? Question "Commercial DT platforms in market provide modelling and simulation alongside integration and UI. DTaas is not able to do any modelling or simulation on its own like other commercial platforms. Is this a correct understanding?" -Yes, you are right + Yes, you are right ??? Question "Can DTaaS support only the information models (or behavioral models) or some other kind of models?" -The DTaaS as such is agnostic to the kind of models you use. DTaaS can -run all kinds of models. This includes behavioral and data models. -As long as you have models and the matching solvers that can run in -Linux OS, you are good to go in DTaaS. -In some cases, models and solvers (tools) are bundled together -to form monolithic DTs. The DTaaS does not limit you from running -such DTs as well. -DTaaS does not provide dedicated solvers. But if you can install -a solver in your workspace, then you don't need the platform -to provide one. + The DTaaS as such is agnostic to the kind of models you use. DTaaS can + run all kinds of models. This includes behavioral and data models. + As long as you have models and the matching solvers that can run in + Linux OS, you are good to go in DTaaS. + In some cases, models and solvers (tools) are bundled together + to form monolithic DTs. The DTaaS does not limit you from running + such DTs as well. + DTaaS does not provide dedicated solvers. But if you can install + a solver in your workspace, then you don't need the platform + to provide one. ??? Question "Does it support XML-based representation and ontology representation?" -Currently No. **We are looking for users needing this capability.** -**If you have concrete requirements and an example, we can discuss a way** -**of realizing it in DTaaS**. + Currently No. **We are looking for users needing this capability.** + **If you have concrete requirements and an example, we can discuss a way** + **of realizing it in DTaaS**. ## Communication Between Physical Twin and Digital Twin ??? Question "How would you measure a physical entity like shape, size, weight, structure, chemical attributes etc. using DTaaS? Any specific technology used in this case?" -The real measurements are done at physical twin which are then -communicated to the digital twin. Any digital twin platform like -DTaaS can only facilitate this communication of these measurements -from physical twin. The DTaaS provides InfluxDB, RabbitMQ and Mosquitto -services for this purpose. These three are probably most widely used -services for digital twin communication. -Having said that, DTaaS allows you to utilize other communication -technologies and services hosted elsewhere on the Internet. + The real measurements are done at physical twin which are then + communicated to the digital twin. Any digital twin platform like + DTaaS can only facilitate this communication of these measurements + from physical twin. The DTaaS provides InfluxDB, RabbitMQ and Mosquitto + services for this purpose. These three are probably most widely used + services for digital twin communication. + Having said that, DTaaS allows you to utilize other communication + technologies and services hosted elsewhere on the Internet. ??? Question "How a real-time data can be differed from static data and what is the procedure to identify dynamic data? Is there any UI or specific tool used here?" -DTaaS can not understand the static or dynamic nature of data. -It can facilitate storing names, units and any other text description -of interesting quantities (weight of batter, voltage output etc). -It can also store the data being sent by the physical twin. -The distinction between static and dynamic data needs to be made by -the user. -Only metadata of the data can reveal such more information about -the nature of data. A tool can probably help in very specific cases, -but you need metadata. If there is a human being making this distinction, -then the need for metadata goes down but does not completely go away. -In some of the DT platforms supported by manufacturers, there is a tight -integration between data and model. In this case, the tool itself is -taking care of the metadata. The DTaaS is a generic platform which can -support execution of digital twins. If a tool can be executed on -a Linux desktop / commandline, the tool can be supported within DTaaS. -The tool (ex. Matlab) itself can take care of the metadata requirements. + DTaaS can not understand the static or dynamic nature of data. + It can facilitate storing names, units and any other text description + of interesting quantities (weight of batter, voltage output etc). + It can also store the data being sent by the physical twin. + The distinction between static and dynamic data needs to be made by + the user. + Only metadata of the data can reveal such more information about + the nature of data. A tool can probably help in very specific cases, + but you need metadata. If there is a human being making this distinction, + then the need for metadata goes down but does not completely go away. + In some of the DT platforms supported by manufacturers, there is a tight + integration between data and model. In this case, the tool itself is + taking care of the metadata. The DTaaS is a generic platform which can + support execution of digital twins. If a tool can be executed on + a Linux desktop / commandline, the tool can be supported within DTaaS. + The tool (ex. Matlab) itself can take care of the metadata requirements. ??? Question "How can DTaaS control the physical entity? Which technologies it uses for controlling the physical world?" -At a very abstract level, there is a communication from physical entity -to digital entity and back to physical entity. How this communication -should happen is decided by the person designing the digital entity. -The DTaaS can provide communication services that can help you do this -communication with relative ease. -You can use InfluxDB, RabbitMQ and Mosquitto services hosted on DTaaS for -two communication between digital and physical entities. + At a very abstract level, there is a communication from physical entity + to digital entity and back to physical entity. How this communication + should happen is decided by the person designing the digital entity. + The DTaaS can provide communication services that can help you do this + communication with relative ease. + You can use InfluxDB, RabbitMQ and Mosquitto services hosted on DTaaS for + two communication between digital and physical entities. ## Data Management ??? Question "Does DTaaS support data collection from different sources like hardware, software and network? Is there any user interface or any tracking instruments used for data collection?" -The DTaaS provids InfluxDB, RabbitMQ, MQTT services. Both the physical twin -and digital twin can utilize these protocols for communication. -The IoT (time-series) data can be collected using InfluxDB and MQTT -broker services. There is a user interface for InfluxDB which can be -used to analyze the data collected. -Users can also manually upload their data files into DTaaS. + The DTaaS provids InfluxDB, RabbitMQ, MQTT services. Both the physical twin + and digital twin can utilize these protocols for communication. + The IoT (time-series) data can be collected using InfluxDB and MQTT + broker services. There is a user interface for InfluxDB which can be + used to analyze the data collected. + Users can also manually upload their data files into DTaaS. ??? Question "Which transmission protocol does DTaaS allow?" -InfluxDB, RabbitMQ, MQTT and anything else that can be used from -Cloud service providers. + InfluxDB, RabbitMQ, MQTT and anything else that can be used from + Cloud service providers. ??? Question "Does DTaaS support multisource information and combined multi sensor input data? Can it provide analysis and decision-supporting inferences?" -You can store information from multiple sources. The existing InfluxDB -services hosted on DTaaS already has a dedicated Influx / Flux query -language for doing sensor fusion, analysis and inferences. + You can store information from multiple sources. The existing InfluxDB + services hosted on DTaaS already has a dedicated Influx / Flux query + language for doing sensor fusion, analysis and inferences. ??? Question "Which kinds of visualization technologies DTaaS can support (e.g. graphical, geometry, image, VR/AR representation)?" -Graphical, geometric and images. If you need specific licensed software -for the visualization, you will have to bring the license for it. -DTaaS does not support AR/VR. + Graphical, geometric and images. If you need specific licensed software + for the visualization, you will have to bring the license for it. + DTaaS does not support AR/VR. ??? Question "Can DTaaS collect data directly from sensors?" -Yes + Yes ??? Question "Is DTaaS able to transmit data to cloud in real time?" -Yes + Yes ## Platform Native Services on DTaaS Platform ??? Question "Is DTaaS able to detect the anomalies about-to-fail components and prescribe solutions?" -This is the job of a digital twin. If you have a ready to use digital twin -that does the job, DTaaS allows others to use your solution. + This is the job of a digital twin. If you have a ready to use digital twin + that does the job, DTaaS allows others to use your solution. ## Comparison with other DT Platforms ??? Question "All the DT platforms seem to provide different features. Is there a comparison chart?" -Here is a qualitative comparison of different DT integration platforms: -Legend: high performance (**H**), mid performance (**M**) -and low performance (**L**) -| DT Platforms | License | DT Development Process | Connectivity | Security | Processing power, performance and Scalability | Data Storage | Visualization | Modeling and Simulation | -|:---|:---|:---|:---|:---|:---|:---|:---|:---| -| Microsoft Azure DT | Commercial Cloud | H | H | H | M | H | H | H | -| AWS IOT Greengrass | Open source commercial | H | H | H | M | H | H | H | -| Eclipse Ditto | Open source | M | H | M | H | H | L | L | -| Asset Administration Shell | Open source | H | H | L | H | M | L | M | -| PTC Thingworx | Commercial | H | H | H | H | H | M | M | -| GE Predix | Commercial | M | H | H | M | L | M | L | -| AU's DTaaS | Open source | H | H | L | L | M | M | M | -Adopted by Tanusree Roy from Table 4 and 5 of the following paper. -Ref: Naseri, F., Gil, S., Barbu, C., Cetkin, E., Yarimca, G., Jensen, A. C., -... & Gomes, C. (2023). Digital twin of electric vehicle battery systems: -Comprehensive review of the use cases, requirements, and platforms. -Renewable and Sustainable Energy Reviews, 179, 113280. + Here is a qualitative comparison of different DT integration platforms: + + Legend: high performance (**H**), mid performance (**M**) + and low performance (**L**) + + | DT Platforms | License | DT Development Process | Connectivity | Security | Processing power, performance and Scalability | Data Storage | Visualization | Modeling and Simulation | + |:---|:---|:---|:---|:---|:---|:---|:---|:---| + | Microsoft Azure DT | Commercial Cloud | H | H | H | M | H | H | H | + | AWS IOT Greengrass | Open source commercial | H | H | H | M | H | H | H | + | Eclipse Ditto | Open source | M | H | M | H | H | L | L | + | Asset Administration Shell | Open source | H | H | L | H | M | L | M | + | PTC Thingworx | Commercial | H | H | H | H | H | M | M | + | GE Predix | Commercial | M | H | H | M | L | M | L | + | AU's DTaaS | Open source | H | H | L | L | M | M | M | + + Adopted by Tanusree Roy from Table 4 and 5 of the following paper. + + Ref: Naseri, F., Gil, S., Barbu, C., Cetkin, E., Yarimca, G., Jensen, A. C., + ... & Gomes, C. (2023). Digital twin of electric vehicle battery systems: + Comprehensive review of the use cases, requirements, and platforms. + Renewable and Sustainable Energy Reviews, 179, 113280. ??? Question "All the comparisons between DT platforms seems so confusing. Why?" -The fundamental confusion comes from the fact that different DT platforms -(Azure DT, GE Predix) provide different kind of DT capabilities. You can -run all kinds of models natively in GE Predix. In fact you can run models -even next to (on) PTs using GE Predix. But you cannot natively do that in -Azure DT service. You have to do the leg work of integrating with other -Azure services or third-party services to get the kind of capabilities -that GE Predix natively provides in one interface. -The takeaway is that we pick horses for the courses. + The fundamental confusion comes from the fact that different DT platforms + (Azure DT, GE Predix) provide different kind of DT capabilities. You can + run all kinds of models natively in GE Predix. In fact you can run models + even next to (on) PTs using GE Predix. But you cannot natively do that in + Azure DT service. You have to do the leg work of integrating with other + Azure services or third-party services to get the kind of capabilities + that GE Predix natively provides in one interface. + The takeaway is that we pick horses for the courses. + + diff --git a/docs/PUBLISH.md b/docs/PUBLISH.md index d77ae3664..c2ac49f4e 100644 --- a/docs/PUBLISH.md +++ b/docs/PUBLISH.md @@ -17,6 +17,18 @@ Install Mkdocs using the following command. pip install -r docs/requirements.txt ``` +## Fix Linting Errors + +This project uses **markdownlint** linter tool for identifying the formatting +issues in markdown files. Run + +```sh +mdl docs +``` + +from top-directory of the project and fix any identified issues. This needs +to be done before committing changes to the documentation. + ## Create documentation The document generation pipeline can generate both **html** and **pdf** diff --git a/docs/admin/client/CLIENT.md b/docs/admin/client/CLIENT.md index 7a611e143..0a4806532 100644 --- a/docs/admin/client/CLIENT.md +++ b/docs/admin/client/CLIENT.md @@ -142,4 +142,4 @@ docker run -d \ Given that multiple services are running at different routes, a reverse proxy is needed to map the background services to external routes. You can use Apache, NGINX, Traefik or any other software to work -as reverse proxy. \ No newline at end of file +as reverse proxy. diff --git a/docs/admin/client/auth.md b/docs/admin/client/auth.md index 995575a5c..ce9f0bb38 100644 --- a/docs/admin/client/auth.md +++ b/docs/admin/client/auth.md @@ -1,4 +1,4 @@ -# OAuth Authentication for React Website +# OAuth Authentication for React Client Website The react client website uses OAuth authentication protocol for user authentication. The PKCE authentication flow of OAuth protocol is used @@ -7,8 +7,10 @@ server. **An oauth application needs to be created on a gitlab instance under admin user**. Then all other users can use the same gitlab instance for oauth authentication. This means commercial gitlab.com can not be used for multi-user authentication -system required by DTaaS. The simplest way to make this work is to setup OAuth -application as +system required by DTaaS. You will need to run an on-premise instance of gitlab. +You can use +[gitlab omnibus docker](https://docs.gitlab.com/ee/install/docker.html) for +this purpose. Please setup OAuth application as [instance wide authentication type](https://docs.gitlab.com/ee/integration/oauth_provider.html#create-an-instance-wide-application). Before setting up oauth application on gitlab, you need to first decide on the @@ -30,34 +32,23 @@ Callback URL: https://foo.com/Library Logout URL: https://foo.com ``` -If you choose to host your DTaaS application with a basename (say bar), then the -URLs change to: - -```txt -DTaaS application URL: https://foo.com/bar -Gitlab instance URL: https://foo.gitlab.com -Callback URL: https://foo.com/bar/Library -Logout URL: https://foo.com/bar -``` - During the creation of oauth application on gitlab, you need to decide on the scope of this oauth application. Choose `openid profile read_user read_repository api` scopes. After successful creation of oauth application, gitlab generates an application ID. This application ID is a long string of HEX values. You need to note this -down and use in configuration files. An example oauth Client ID is: `934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0`. +down and use in configuration files. An example oauth Client ID is: +`934b98f03f1b6f743832b2840bf7cccaed93c3bfe579093dd0942a433691ccc0`. -The mapping between the oauth URLs and the environment variables in `env.js` -is shown below. +You need the following information from the OAuth application registered on Gitlab: -| URL | Variable name in env.js | -|:---|:---| -| DTaaS application URL | REACT_APP_URL | -| Gitlab instance URL | REACT_APP_AUTH_AUTHORITY | -| Callback URL | REACT_APP_REDIRECT_URI | -| Logout URL | REACT_APP_LOGOUT_REDIRECT_URI | -|| +| Gitlab Variable Name | Variable name in Client env.js | Default Value | +|:---|:---|:---| +| OAuth Provider | REACT_APP_AUTH_AUTHORITY | https://gitlab.foo.com/ | +| Application ID | REACT_APP_CLIENT_ID | +| Callback URL | REACT_APP_REDIRECT_URI | https://foo.com/Library | +| Scopes | REACT_APP_GITLAB_SCOPES | openid, profile, read_user, read_repository, api | The same **URLs** and **Client ID** are useful for both the regular hosting of DTaaS application and also as the CI/CD server to be used for the development work. @@ -103,4 +94,14 @@ All of these instances can use the same gitlab instance for authentication. If you are hosting multiple DTaaS instances on the same server, do not a DTaaS with a null basename on the same server. Even though it works well, the setup is confusing to setup -and may lead to maintenance issues. \ No newline at end of file +and may lead to maintenance issues. + +If you choose to host your DTaaS application with a basename (say bar), then the +URLs in `env.js` change to: + +```txt +DTaaS application URL: https://foo.com/bar +Gitlab instance URL: https://foo.gitlab.com +Callback URL: https://foo.com/bar/Library +Logout URL: https://foo.com/bar +``` diff --git a/docs/admin/host.md b/docs/admin/host.md index be2661f44..318bf732f 100644 --- a/docs/admin/host.md +++ b/docs/admin/host.md @@ -1,7 +1,7 @@ # DTaaS on Linux Operating System These are installation instructions for running DTaaS application -on a Ubuntu Server 20.04 Operating System. +on a Ubuntu Server 22.04 Operating System. The setup requires a machine which can spare 16GB RAM, 8 vCPUs and 50GB Hard Disk space. @@ -41,10 +41,19 @@ Change `foo.com` to your local hostname and user1/user2 to the usernames chosen by you. !!! tip - Do not use `http://` or `https://` in _deploy/config/gateway/fileConfig.yml_. + Do not use `http://` or `https://` + in _deploy/config/gateway/fileConfig.yml_. #### Authentication +This step requires `htpasswd` commandline utility. If +it is not available on your system, please install the same by using + +```bash +sudo apt-get install -y apache2-utils +``` + +You can now proceed with update of the gateway authentication setup. The dummy username is `foo` and the password is `bar`. Please change this before starting the gateway. @@ -82,7 +91,28 @@ APOLLO_PATH='/lib' GRAPHQL_PLAYGROUND='true' ``` -## Configure react website +## Configure React Client Website + +### Gitlab OAuth application + +The DTaaS react website requires Gitlab OAuth provider. +If you need more help with this step, please see +the [Authentication page](client/auth.md). + +You need the following information from the OAuth application registered on Gitlab: + +| Gitlab Variable Name | Variable name in Client env.js | Default Value | +|:---|:---|:---| +| OAuth Provider | REACT_APP_AUTH_AUTHORITY | https://gitlab.foo.com/ | +| Application ID | REACT_APP_CLIENT_ID | +| Callback URL | REACT_APP_REDIRECT_URI | https://foo.com/Library | +| Scopes | REACT_APP_GITLAB_SCOPES | openid, profile, read_user, read_repository, api | + +You can also see +[Gitlab help page](https://docs.gitlab.com/ee/integration/oauth_provider.html) +for getting the Gitlab OAuth application details. + +### Update Client Config Change the React website configuration in _deploy/config/client/env.js_. diff --git a/docs/admin/services.md b/docs/admin/services.md index a96ed4fd8..96362b054 100644 --- a/docs/admin/services.md +++ b/docs/admin/services.md @@ -15,24 +15,20 @@ multiple hostnames for different services. You can download the required services using the docker commands. ```sh -docker pull traefik:v2.5 -docker pull influxdb:2.4 -docker pull mltooling/ml-workspace:0.13.2 -docker pull grafana/grafana -docker pull gitlab/gitlab-ce:15.10.0-ce.0 +docker pull grafana/grafana:10.1.4 +docker pull influxdb:2.7 +docker pull telegraf:1.28.2 +docker pull rabbitmq:3-management +docker pull eclipse-mosquitto:2 ``` -:warning: The docker images are large and are likely to consume -about 5GB of bandwidth and 15GB of space. -You will have to download the docker images on a really good network. - The two-machine vagrant deployment scenario installs the RabbitMQ, Grafana, and InfluxDB services on the second vagrant machine. If you would like to install some of these services for native OS installation or single vagrant machine, you can do this as well. -## RabbitMQ Service +## RabbitMQ Start the RabbitMQ service with @@ -60,7 +56,7 @@ TCP/UDP traffic. There are two possible choices here: Unless you are an informed user of Traefik, we recommend bypassing traefik for RabbitMQ service. -## Grafana Service +## Grafana Grafana service can run well behind Traefik gateway. Here is a sample docker command to run Grafana service at port 3000: @@ -96,11 +92,12 @@ printf "Complete the setup from GUI" The user credentials have also been set in the command as: **username**: admin + **password**: DTaaSGrafana Remember to change these credentials before starting the docker container. -## InfluxDB Service +## InfluxDB The barebones InfluxDB service can be installed using: @@ -124,6 +121,7 @@ docker run -d -p 80:8086 \ The user credentials have also been set in the command as: **username**: dtaas + **password**: dtaas1357 Remember to change these credentials before starting the docker container. diff --git a/docs/admin/trial.md b/docs/admin/trial.md index 55da68e38..aee3eefc9 100644 --- a/docs/admin/trial.md +++ b/docs/admin/trial.md @@ -6,11 +6,41 @@ on vagrant virtual machine(s). A single step install script is helpful in performing a trial run of the software. This script installs DTaaS software -with default credentials and users on a Ubuntu server OS. -You can use it to check a test installation of DTaaS. +with default credentials and users on a Ubuntu server 22.04 +Operating System. +You can use it to check a test installation of DTaaS software. + +## Pre-requisites + +### Domain name + +You need a domain name to run the application. The install script +assumes **foo.com** to be your domain name. Please change it +to a relevant one. + +### Gitlab OAuth application + +The DTaaS react website requires Gitlab OAuth provider. +If you need more help with this step, please see +the [Authentication page](client/auth.md). + +You need the following information from the OAuth application registered on Gitlab: + +| Gitlab Variable Name | Variable name in Client env.js | Default Value | +|:---|:---|:---| +| OAuth Provider | REACT_APP_AUTH_AUTHORITY | https://gitlab.foo.com/ | +| Application ID | REACT_APP_CLIENT_ID | +| Callback URL | REACT_APP_REDIRECT_URI | https://foo.com/Library | +| Scopes | REACT_APP_GITLAB_SCOPES | openid, profile, read_user, read_repository, api | + +You can also see +[Gitlab help page](https://docs.gitlab.com/ee/integration/oauth_provider.html) +for getting the Gitlab OAuth application details. + +### Install ```bash -wget https://github.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/single-script-install.sh +wget https://raw.githubusercontent.com/INTO-CPS-Association/DTaaS/feature/distributed-demo/deploy/single-script-install.sh bash single-script-install.sh ``` diff --git a/docs/admin/vagrant/base-box.md b/docs/admin/vagrant/base-box.md index 03de02ca6..23ffccd42 100644 --- a/docs/admin/vagrant/base-box.md +++ b/docs/admin/vagrant/base-box.md @@ -1,32 +1,63 @@ -# Vagrant Box for DTaaS +# DTaaS Vagrant Box -There are some good vagrant boxes on -[vagrant website](https://app.vagrantup.com/boxes/search). -But these boxes require too many installations that take a -long time and network bandwidth. So it is efficient to -create one local vagrant box for DTaaS application and reuse -the same in all installations. +This README provides instructions on creating a custom Operating System +virtual disk for running the DTaaS software. The virtual disk is managed +by **vagrant**. The purpose is two fold: -## Installed Software +* Provide cross-platform installation of the DTaaS application. + Any operating system supporting use of vagrant software utility + can support installation of the DTaaS software. +* Create a ready to use development environment for code contributors. -This base DTaaS vagrant box, when it is successfully created, has the following software: +There are two scripts in this directory: -* docker -* nodejs and yarn -* jupyter -* microk8s -* containers - * mltooling/ml-workspace:0.13.2 - * traefik2.5 - * influxdb2.4 - * grafana - * telegraf - * gitlab +| Script name | Purpose | Default | +|:---|:---|:---| +| `user.sh` | user installation | :white_check_mark: | +| `developer.sh` | developer installation | :x: | + +If you are installing the DTaaS for developers, the default installation +caters to your needs. You can skip the next step and continue with the +creation of vagrant box. + +If you are a developer and would like additional software installed, you need +to modify `Vagrantfile`. The existing `Vagrantfile` has two lines: + +```sh + config.vm.provision "shell", path: "user.sh" + #config.vm.provision "shell", path: "developer.sh" +``` -## Create the vagrant box +Uncomment the second line to have more software components installed. If you +are not a developer, no changes are required to the `Vagrantfile`. + +This vagrant box installed for users will have the following items: + +* docker v24.0 +* nodejs v18.8 +* yarn v1.22 +* npm v10.2 +* containers + * ml-workspace v0.13 + * traefik v2.10 + * gitlab-ce v16.4 + * influxdb v2.7 + * grafana v10.1 + * rabbitmq v3-management + * eclipse-mosquitto (mqtt) v2 + +This vagrant box installed for developers will have +the following items additional items: + +* docker-compose v2.20 +* microk8s v1.27 +* jupyterlab +* mkdocs +* containers + * telegraf v1.28 -Publish a base virtualbox package to be used by -vagrant to publish all other virtualbox packages +The upcoming instructions will help with the creation of +base vagrant box. ```bash #create a key pair diff --git a/docs/developer/system/current-status.md b/docs/developer/system/current-status.md index dd84e03f8..ce5d3d98b 100644 --- a/docs/developer/system/current-status.md +++ b/docs/developer/system/current-status.md @@ -70,7 +70,7 @@ They are: [Grafana](https://github.com/grafana/grafana), [RabbitMQ](https://github.com/rabbitmq/rabbitmq-server) and -[MQTT](https://mqtt.org/). +[MQTT](https://github.com/eclipse/mosquitto). These services can be used by DTs and PTs for communication, storing and visualization of data. There can also be monitoring services setup diff --git a/docs/index.md b/docs/index.md index f11b7c38f..52daafde2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,6 +28,8 @@ and is available under [the INTO-CPS License](LICENSE.md). The DTaaS software platform uses [Træfik](https://github.com/traefik/traefik), [ML Workspace](https://github.com/ml-tooling/ml-workspace), [Grafana](https://github.com/grafana/grafana), -[InfluxDB](https://github.com/influxdata/influxdb) and -[RabbitMQ](https://github.com/rabbitmq/rabbitmq-server) open-source components. +[InfluxDB](https://github.com/influxdata/influxdb), +[MQTT](https://github.com/eclipse/mosquitto) and +[RabbitMQ](https://github.com/rabbitmq/rabbitmq-server) +open-source components. These software components have their own licenses. diff --git a/docs/thanks.md b/docs/thanks.md index 6374a4cd3..f6872380b 100644 --- a/docs/thanks.md +++ b/docs/thanks.md @@ -12,8 +12,9 @@ Henrik Ejersbo, Tanusree Roy, Farshid Naseri 1. Talasila, P., Gomes, C., Mikkelsen, P. H., Arboleda, S. G., Kamburjan, E., & Larsen, P. G. (2023). - [Digital Twin as a Service (DTaaS): A Platform for Digital Twin Developers and Users](https://arxiv.org/abs/2305.07244). - arXiv preprint arXiv:2305.07244. + Digital Twin as a Service (DTaaS): A Platform for + Digital Twin Developers and Users + [arXiv preprint arXiv:2305.07244](https://arxiv.org/abs/2305.07244). 1. Astitva Sehgal 1. Tanusree Roy -1. Farshid Naseri \ No newline at end of file +1. Farshid Naseri diff --git a/docs/user/digital-twins/create.md b/docs/user/digital-twins/create.md index ffe48b663..119e26da1 100644 --- a/docs/user/digital-twins/create.md +++ b/docs/user/digital-twins/create.md @@ -136,8 +136,8 @@ The [lifecycle page](lifecycle.md) provides more explanation on these programs. [example files](https://github.com/INTO-CPS-Association/DTaaS-examples/archive/refs/heads/main.zip) ```sh - $wget https://github.com/INTO-CPS-Association/DTaaS-examples/archive/refs/heads/main.zip - $unzip main.zip + wget https://github.com/INTO-CPS-Association/DTaaS-examples/archive/refs/heads/main.zip + unzip main.zip ``` 1. Open a file browser and copy the files from this uncompressed folder @@ -145,9 +145,9 @@ The [lifecycle page](lifecycle.md) provides more explanation on these programs. matches the one given above. 1. Go to the digital twin directory and run - ``` - $cd /workspace/digital twins/mass-spring-damper - $lifecycle/execute + ```sh + cd /workspace/digital twins/mass-spring-damper + lifecycle/execute ``` The last command executes the mass-spring-damper digital twin and stores diff --git a/docs/user/examples/index.md b/docs/user/examples/index.md index 799d65a43..b2b6afad2 100644 --- a/docs/user/examples/index.md +++ b/docs/user/examples/index.md @@ -1,3 +1,4 @@ # DTaaS examples -The examples are hosted in [DTaaS examples repository](https://github.com/INTO-CPS-Association/DTaaS-examples) \ No newline at end of file +The examples are hosted in +[DTaaS examples repository](https://github.com/INTO-CPS-Association/DTaaS-examples) diff --git a/docs/user/servers/lib/LIB-MS.md b/docs/user/servers/lib/LIB-MS.md index 98702993b..044db5f00 100644 --- a/docs/user/servers/lib/LIB-MS.md +++ b/docs/user/servers/lib/LIB-MS.md @@ -145,16 +145,16 @@ send requests to: https://foo.com/lib === "HTTP Response" -``` http-response -HTTP/1.1 200 OK -Access-Control-Allow-Origin: * -Connection: close -Content-Length: 306 -Content-Type: application/json; charset=utf-8 -Date: Tue, 26 Sep 2023 20:26:49 GMT -X-Powered-By: Express -{"data":{"listDirectory":{"repository":{"tree":{"blobs":{"edges":[]},"trees":{"edges":[{"node":{"name":"data","type":"tree"}},{"node":{"name":"digital twins","type":"tree"}},{"node":{"name":"functions","type":"tree"}},{"node":{"name":"models","type":"tree"}},{"node":{"name":"tools","type":"tree"}}]}}}}}} -``` + ``` http-response + HTTP/1.1 200 OK + Access-Control-Allow-Origin: * + Connection: close + Content-Length: 306 + Content-Type: application/json; charset=utf-8 + Date: Tue, 26 Sep 2023 20:26:49 GMT + X-Powered-By: Express + {"data":{"listDirectory":{"repository":{"tree":{"blobs":{"edges":[]},"trees":{"edges":[{"node":{"name":"data","type":"tree"}},{"node":{"name":"digital twins","type":"tree"}},{"node":{"name":"functions","type":"tree"}},{"node":{"name":"models","type":"tree"}},{"node":{"name":"tools","type":"tree"}}]}}}}}} + ``` ### Fetch a file from the available files @@ -165,68 +165,68 @@ with content of `hello world`. === "GraphQL Request" -```graphql-request -query { - readFile(path: "user2/data/sample.txt") { - repository { - blobs { - nodes { - name - rawBlob - rawTextBlob + ```graphql-request + query { + readFile(path: "user2/data/sample.txt") { + repository { + blobs { + nodes { + name + rawBlob + rawTextBlob + } + } } } } - } -} -``` + ``` === "GraphQL Response" -```graphql-response -{ - "data": { - "readFile": { - "repository": { - "blobs": { - "nodes": [ - { - "name": "sample.txt", - "rawBlob": "hello world", - "rawTextBlob": "hello world" + ```graphql-response + { + "data": { + "readFile": { + "repository": { + "blobs": { + "nodes": [ + { + "name": "sample.txt", + "rawBlob": "hello world", + "rawTextBlob": "hello world" + } + ] } - ] + } } } } - } -} -``` + ``` === "HTTP Request" -```http-request -POST /lib HTTP/1.1 -Host: foo.com -Content-Type: application/json -Content-Length: 217 -{ - "query":"query {\n readFile(path: \"user2/data/welcome.txt\") {\n repository {\n blobs {\n nodes {\n name\n rawBlob\n rawTextBlob\n }\n }\n }\n }\n}" -} -``` + ```http-request + POST /lib HTTP/1.1 + Host: foo.com + Content-Type: application/json + Content-Length: 217 + { + "query":"query {\n readFile(path: \"user2/data/welcome.txt\") {\n repository {\n blobs {\n nodes {\n name\n rawBlob\n rawTextBlob\n }\n }\n }\n }\n}" + } + ``` === "HTTP Response" -```http-response -HTTP/1.1 200 OK -Access-Control-Allow-Origin: * -Connection: close -Content-Length: 134 -Content-Type: application/json; charset=utf-8 -Date: Wed, 27 Sep 2023 09:17:18 GMT -X-Powered-By: Express -{"data":{"readFile":{"repository":{"blobs":{"nodes":[{"name":"welcome.txt","rawBlob":"hello world","rawTextBlob":"hello world"}]}}}}} -``` + ```http-response + HTTP/1.1 200 OK + Access-Control-Allow-Origin: * + Connection: close + Content-Length: 134 + Content-Type: application/json; charset=utf-8 + Date: Wed, 27 Sep 2023 09:17:18 GMT + X-Powered-By: Express + {"data":{"readFile":{"repository":{"blobs":{"nodes":[{"name":"welcome.txt","rawBlob":"hello world","rawTextBlob":"hello world"}]}}}}} + ``` The _path_ refers to the file path to look at: For example, _user1_ looks at files of diff --git a/script/docker.sh b/script/docker.sh index ffa5bd2ff..8d10fbd47 100644 --- a/script/docker.sh +++ b/script/docker.sh @@ -1,8 +1,10 @@ #!/bin/bash # get the required docker images -docker pull traefik:v2.5 -docker pull influxdb:2.4 +docker pull traefik:v2.10 docker pull mltooling/ml-workspace:0.13.2 -docker pull grafana/grafana -docker pull telegraf -docker pull gitlab/gitlab-ce:15.10.0-ce.0 +docker pull grafana/grafana:10.1.4 +docker pull influxdb:2.7 +docker pull telegraf:1.28.2 +docker pull rabbitmq:3-management +docker pull eclipse-mosquitto:2 +docker pull gitlab/gitlab-ce:16.4.1-ce.0 \ No newline at end of file diff --git a/script/env.sh b/script/env.sh index 8c853701a..5135ecdef 100755 --- a/script/env.sh +++ b/script/env.sh @@ -12,13 +12,15 @@ sudo apt-get install -y \ lsb-release \ zsh \ apache2-utils \ - net-tools + net-tools \ + python3-dev \ + python3-pip sudo mkdir -p /etc/apt/keyrings if [ ! -f /etc/apt/keyrings/docker.gpg ] then curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - sudo printf \ + printf \ "deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ %s stable" "$(dpkg --print-architecture)" "$(lsb_release -cs)" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null fi @@ -51,15 +53,38 @@ sudo systemctl enable docker.service sudo systemctl enable containerd.service +#install docker-compose from https://docs.docker.com/compose/install/other/ +sudo curl -SL "https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64" \ + -o /usr/local/bin/docker-compose +sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose +sudo chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose + # Install nodejs environment -curl -fsSL https://deb.nodesource.com/setup_18.x | sudo bash - +sudo apt-get update +sudo apt-get install -y ca-certificates curl gnupg +sudo mkdir -p /etc/apt/keyrings +if [ ! -f /etc/apt/keyrings/nodesource.gpg ] +then + curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \ + sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +fi +NODE_MAJOR=18 +printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \ + https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \ + sudo tee /etc/apt/sources.list.d/nodesource.list + +sudo apt-get update sudo apt-get install -y nodejs +sudo npm install -g npm@10.2.0 + if [ ! -f /usr/share/keyrings/yarnkey.gpg ] then - curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null - printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main" | sudo tee /etc/apt/sources.list.d/yarn.list + curl -sL "https://dl.yarnpkg.com/debian/pubkey.gpg" | gpg --dearmor | \ + sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null + printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main \n" | \ + sudo tee /etc/apt/sources.list.d/yarn.list fi sudo apt-get update -y @@ -71,11 +96,11 @@ sudo apt-get install -y wget openssl # Install playwright tool for integration tests on browsers -npx playwright install-deps +npx --yes playwright install-deps #------------- printf "\n\n Installing required python packages...." -apt install -y python3-pip +sudo apt install -y python3-pip sudo -H pip3 install mkdocs sudo -H pip3 install mkdocs-material sudo -H pip3 install python-markdown-math @@ -84,8 +109,9 @@ sudo -H pip3 install mkdocs-with-pdf sudo -H pip3 install qrcode # Install markdownlint +sudo apt-get install -y rubygems sudo gem install mdl # Install madge for generating dependency graphs of typescript projects -sudo apt-get install graphviz +sudo apt-get install -y graphviz sudo npm install -g madge \ No newline at end of file diff --git a/servers/config/gateway/README.md b/servers/config/gateway/README.md index dfb3159a7..432155277 100644 --- a/servers/config/gateway/README.md +++ b/servers/config/gateway/README.md @@ -25,7 +25,7 @@ docker run -d \ --network=host -v $PWD/traefik.yml:/etc/traefik/traefik.yml \ -v $PWD/dynamic:/etc/traefik/dynamic \ -v /var/run/docker.sock:/var/run/docker.sock \ -traefik:v2.5 +traefik:v2.10 ``` ## Authentication diff --git a/servers/execution/runner/script/clean.bash b/servers/execution/runner/script/clean.bash index 14ec423c5..1d6cbdcea 100755 --- a/servers/execution/runner/script/clean.bash +++ b/servers/execution/runner/script/clean.bash @@ -1,2 +1,2 @@ #!/bin/bash -rm -rf build node_modules coverage dist *.svg \ No newline at end of file +rm -rf build node_modules coverage dist "*.svg" \ No newline at end of file diff --git a/servers/lib/script/clean.bash b/servers/lib/script/clean.bash index 43edac9e4..a11e0e0f6 100755 --- a/servers/lib/script/clean.bash +++ b/servers/lib/script/clean.bash @@ -1,2 +1,2 @@ #!/bin/bash -rm -rf build node_modules coverage dist *.svg \ No newline at end of file +rm -rf build node_modules coverage dist "*.svg" \ No newline at end of file diff --git a/servers/lib/test/starttraefik.bash b/servers/lib/test/starttraefik.bash index a9be47162..e1cae462a 100755 --- a/servers/lib/test/starttraefik.bash +++ b/servers/lib/test/starttraefik.bash @@ -7,4 +7,4 @@ docker run -d \ --network=host -v "$PWD/traefik.yml":/etc/traefik/traefik.yml \ -v "$PWD/dynamic":/etc/traefik/dynamic \ -v /var/run/docker.sock:/var/run/docker.sock \ -traefik:v2.5 \ No newline at end of file +traefik:v2.10 \ No newline at end of file