Skip to content

Data are loaded on the globalworker too. #1904

Data are loaded on the globalworker too.

Data are loaded on the globalworker too. #1904

name: Production Env Tests
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
run_tests:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@v3
- name: Set up python
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install Poetry
uses: snok/install-poetry@v1
with:
version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102
virtualenvs-create: true
virtualenvs-in-project: true
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}
- name: Install dependencies
if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
run: poetry install --no-interaction --no-root
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Load MONETDB cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/monetdb
key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
restore-keys: |
${{ runner.os }}-buildx-3-monetdb-
- name: Build MONETDB docker image
uses: docker/build-push-action@v3
with:
context: .
file: monetdb/Dockerfile
push: false
load: true
tags: madgik/exareme2_db:dev
cache-from: type=local,src=/tmp/.buildx-cache/monetdb
cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb
- name: Load MIPDB container cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/mipdb
key: ${{ runner.os }}-buildx-mipdb-${{hashFiles('mipdb/**')}}
restore-keys: |
${{ runner.os }}-buildx-mipdb-
- name: Build MIPDB container docker image
uses: docker/build-push-action@v3
with:
context: .
file: mipdb/Dockerfile
push: false
load: true
tags: madgik/exareme2_mipdb:dev
cache-from: type=local,src=/tmp/.buildx-cache/mipdb
cache-to: type=local,dest=/tmp/.buildx-cache-new/mipdb
- name: Load RABBITMQ cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/rabbitmq
key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
restore-keys: |
${{ runner.os }}-buildx-rabbitmq-
- name: Build RABBITMQ docker image
uses: docker/build-push-action@v3
with:
context: .
file: rabbitmq/Dockerfile
push: false
load: true
tags: madgik/exareme2_rabbitmq:dev
cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq
- name: Load WORKER service cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/worker
key: ${{ runner.os }}-buildx-worker-${{hashFiles('exareme2/**')}}
restore-keys: |
${{ runner.os }}-buildx-worker-
- name: Build WORKER service docker image
uses: docker/build-push-action@v3
with:
context: .
file: exareme2/worker/Dockerfile
push: false
load: true
tags: madgik/exareme2_worker:dev
cache-from: type=local,src=/tmp/.buildx-cache/worker
cache-to: type=local,dest=/tmp/.buildx-cache-new/worker
- name: Load CONTROLLER service cached image
uses: actions/cache@v3
with:
path: /tmp/.buildx-cache/controller
key: ${{ runner.os }}-buildx-controller-${{hashFiles('exareme2/**')}}
restore-keys: |
${{ runner.os }}-buildx-controller-
- name: Build CONTROLLER service docker image
uses: docker/build-push-action@v3
with:
context: .
file: exareme2/controller/Dockerfile
push: false
load: true
tags: madgik/exareme2_controller:dev
cache-from: type=local,src=/tmp/.buildx-cache/controller
cache-to: type=local,dest=/tmp/.buildx-cache-new/controller
# Temp fix
# https://github.com/docker/build-push-action/issues/252
# https://github.com/moby/buildkit/issues/1896
- name: Move Docker images cache
run: |
rm -rf /tmp/.buildx-cache
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
- name: Create k8s Kind Cluster
uses: helm/[email protected]
with:
cluster_name: kind
config: tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml
- name: Install Helm
uses: azure/setup-helm@v3
with:
version: 3.9.1
id: install
- name: Taint Nodes
run: |
kubectl taint nodes master node-role.kubernetes.io/control-plane-
kubectl label node master master=true
kubectl label node localworker1 worker=true
kubectl label node localworker2 worker=true
- name: Get container disk space
run: df -h
- name: Free up space, by removing dotnet, android and haskell unused libs.
run: |
rm -rf /usr/share/dotnet
rm -rf /opt/ghc
sudo rm -rf /usr/local/lib/android
- name: Get container disk space
run: df -h
- name: Load docker images to kind containers and delete them locally
run: |
kind load docker-image madgik/exareme2_worker:dev
docker image rm madgik/exareme2_worker:dev
kind load docker-image madgik/exareme2_controller:dev --nodes kind-control-plane
docker image rm madgik/exareme2_controller:dev
kind load docker-image madgik/exareme2_db:dev
docker image rm madgik/exareme2_db:dev
kind load docker-image madgik/exareme2_mipdb:dev
docker image rm madgik/exareme2_mipdb:dev
kind load docker-image madgik/exareme2_rabbitmq:dev
docker image rm madgik/exareme2_rabbitmq:dev
- name: Get container disk space
run: df -h
- name: Copy prod_env_tests values.yaml
run: cp -r tests/prod_env_tests/deployment_configs/kubernetes_values.yaml kubernetes/values.yaml
- name: Print Helm Templates
run: helm template kubernetes/
- name: Deploy Helm
run: helm install exareme2 kubernetes/ --debug
- name: Wait for pods to get healthy
run: timeout 300 bash -c 'while true; do if kubectl get pods --no-headers | awk '\''{if ($2 != "1/1" && $2 != "2/2" && $2 != "3/3" && $2 != "4/4") exit 1;}'\''; then echo "All pods are ready!"; break; else kubectl get pods -o wide; sleep 20; fi done'
- name: Initialize MONETDB from mipdb container
run: |
for i in 0 1; do
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[$i].metadata.name")
kubectl exec $POD_NAME -c db-importer -- sh -c 'mipdb init'
done
GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb init'
- name: Load dementia data model into localworkers and globalworker
run: |
for i in 0 1; do
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[$i].metadata.name")
kubectl exec $POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/dementia_v_0_1/CDEsMetadata.json'
done
GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/dementia_v_0_1/CDEsMetadata.json'
- name: Load dementia dataset csvs into localworkers and globalworker
run: |
for suffix in {0..9}; do
if [ $suffix -eq 0 ] || [ $suffix -eq 3 ] || [ $suffix -eq 6 ]; then
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[0].metadata.name")
elif [ $suffix -eq 1 ] || [ $suffix -eq 4 ] || [ $suffix -eq 7 ]; then
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name")
else
POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
fi
for dataset in edsd ppmi desd-synthdata; do
kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/dementia_v_0_1/${dataset}${suffix}.csv -d dementia -v 0.1"
done
done
- name: Load tbi data model into localworkers and globalworker
run: |
for i in 0 1; do
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[$i].metadata.name")
kubectl exec $POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/tbi_v_0_1/CDEsMetadata.json'
done
GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/tbi_v_0_1/CDEsMetadata.json'
- name: Load tbi dataset csvs into localworkers and globalworker
run: |
for suffix in {0..9}; do
if [ $suffix -eq 0 ] || [ $suffix -eq 3 ] || [ $suffix -eq 6 ]; then
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[0].metadata.name")
elif [ $suffix -eq 1 ] || [ $suffix -eq 4 ] || [ $suffix -eq 7 ]; then
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name")
else
POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
fi
kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/tbi_v_0_1/dummy_tbi${suffix}.csv -d tbi -v 0.1"
done
- name: Load longitudinal dementia data model into localworkers and globalworker
run: |
for i in 0 1; do
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[$i].metadata.name")
kubectl exec $POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/longitudinal_dementia_v_0_1/CDEsMetadata.json'
done
GLOBAL_POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
kubectl exec $GLOBAL_POD_NAME -c db-importer -- sh -c 'mipdb add-data-model /opt/data/longitudinal_dementia_v_0_1/CDEsMetadata.json'
- name: Load longitudinal dementia datasets into localworkers and globalworker
run: |
for suffix in {0..2}; do
if [ $suffix -eq 0 ]; then
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[0].metadata.name")
elif [ $suffix -eq 1 ]; then
POD_NAME=$(kubectl get pods -l=nodeType=localworker -o json | jq -r ".items[1].metadata.name")
else
POD_NAME=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r '.items[0].metadata.name')
fi
kubectl exec $POD_NAME -c db-importer -- sh -c "mipdb add-dataset /opt/data/longitudinal_dementia_v_0_1/longitudinal_dementia${suffix}.csv -d longitudinal_dementia -v 0.1"
done
- name: Controller logs
run: kubectl logs -l app=exareme2-controller --tail -1
- name: Globalnode logs
run: kubectl logs -l nodeType=globalworker -c worker --tail -1
- name: Localnode logs
run: kubectl logs -l nodeType=localworker -c worker --tail -1
- name: Controller logs (post run)
uses: webiny/[email protected]
with:
run: kubectl logs -l app=exareme2-controller --tail -1
- name: Globalnode logs (post run)
uses: webiny/[email protected]
with:
run: kubectl logs -l nodeType=globalworker -c worker --tail -1
- name: Localnode logs (post run)
uses: webiny/[email protected]
with:
run: kubectl logs -l nodeType=localworker -c worker --tail -1
- name: Run Worker Landscape Aggregator update
run: curl -X POST "http://172.17.0.1:5000/wla"
- name: Run Healthcheck
run: curl "http://172.17.0.1:5000/healthcheck"
- name: Update WorkerLandscapeAggregator
run: curl -X POST "http://172.17.0.1:5000/wla"
- name: Run production env tests
run: poetry run pytest tests/prod_env_tests --verbosity=4