prod_env_tests.yml

name: Production Env Tests

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  run_tests:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v3

      - name: Set up python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.3.2 # TODO https://github.com/pgjones/hypercorn/issues/102
          virtualenvs-create: true
          virtualenvs-in-project: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ hashFiles('poetry.lock') }}

      - name: Install dependencies
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Load MONETDB cached image
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache/monetdb
          key: ${{ runner.os }}-buildx-3-monetdb-${{hashFiles('monetdb/**')}}-${{ hashFiles('exareme2/udfgen/udfio.py')}}
          restore-keys: |
            ${{ runner.os }}-buildx-3-monetdb-

      - name: Build MONETDB docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: monetdb/Dockerfile
          push: false
          load: true
          tags: madgik/exareme2_db:dev
          cache-from: type=local,src=/tmp/.buildx-cache/monetdb
          cache-to: type=local,dest=/tmp/.buildx-cache-new/monetdb

      - name: Load MIPDB container cached image
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache/mipdb
          key: ${{ runner.os }}-buildx-mipdb-${{hashFiles('mipdb/**')}}
          restore-keys: |
            ${{ runner.os }}-buildx-mipdb-

      - name: Build MIPDB container docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: mipdb/Dockerfile
          push: false
          load: true
          tags: madgik/exareme2_mipdb:dev
          cache-from: type=local,src=/tmp/.buildx-cache/mipdb
          cache-to: type=local,dest=/tmp/.buildx-cache-new/mipdb

      - name: Load RABBITMQ cached image
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache/rabbitmq
          key: ${{ runner.os }}-buildx-rabbitmq-${{hashFiles( 'rabbitmq/**' )}}
          restore-keys: |
            ${{ runner.os }}-buildx-rabbitmq-

      - name: Build RABBITMQ docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: rabbitmq/Dockerfile
          push: false
          load: true
          tags: madgik/exareme2_rabbitmq:dev
          cache-from: type=local,src=/tmp/.buildx-cache/rabbitmq
          cache-to: type=local,dest=/tmp/.buildx-cache-new/rabbitmq

      - name: Load WORKER service cached image
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache/worker
          key: ${{ runner.os }}-buildx-worker-${{hashFiles('exareme2/**')}}
          restore-keys: |
            ${{ runner.os }}-buildx-worker-

      - name: Build WORKER service docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: exareme2/worker/Dockerfile
          push: false
          load: true
          tags: madgik/exareme2_worker:dev
          cache-from: type=local,src=/tmp/.buildx-cache/worker
          cache-to: type=local,dest=/tmp/.buildx-cache-new/worker

      - name: Load CONTROLLER service cached image
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache/controller
          key: ${{ runner.os }}-buildx-controller-${{hashFiles('exareme2/**')}}
          restore-keys: |
            ${{ runner.os }}-buildx-controller-

      - name: Build CONTROLLER service docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          file: exareme2/controller/Dockerfile
          push: false
          load: true
          tags: madgik/exareme2_controller:dev
          cache-from: type=local,src=/tmp/.buildx-cache/controller
          cache-to: type=local,dest=/tmp/.buildx-cache-new/controller

        # Temp fix
        # https://github.com/docker/build-push-action/issues/252
        # https://github.com/moby/buildkit/issues/1896
      - name: Move Docker images cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

      - name: Create k8s Kind Cluster
        uses: helm/kind-action@v1.5.0
        with:
          cluster_name: kind
          config: tests/prod_env_tests/deployment_configs/kind_configuration/kind_cluster.yaml

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: 3.9.1
        id: install

      - name: Taint Nodes
        run: |
          kubectl taint nodes master node-role.kubernetes.io/control-plane-
          kubectl label node master master=true
          kubectl label node localworker1 worker=true
          kubectl label node localworker2 worker=true

      - name: Get container disk space
        run: df -h

      - name: Free up space, by removing dotnet, android and haskell unused libs.
        run: |
          rm -rf /usr/share/dotnet
          rm -rf /opt/ghc
          sudo rm -rf /usr/local/lib/android

      - name: Get container disk space
        run: df -h

      - name: Load docker images to kind containers and delete them locally
        run: |
          kind load docker-image madgik/exareme2_worker:dev
          docker image rm madgik/exareme2_worker:dev
          kind load docker-image madgik/exareme2_controller:dev --nodes kind-control-plane
          docker image rm madgik/exareme2_controller:dev
          kind load docker-image madgik/exareme2_db:dev
          docker image rm madgik/exareme2_db:dev
          kind load docker-image madgik/exareme2_mipdb:dev
          docker image rm madgik/exareme2_mipdb:dev
          kind load docker-image madgik/exareme2_rabbitmq:dev
          docker image rm madgik/exareme2_rabbitmq:dev

      - name: Get container disk space
        run: df -h

      - name: Copy prod_env_tests values.yaml
        run: cp -r tests/prod_env_tests/deployment_configs/kubernetes_values.yaml kubernetes/values.yaml

      - name: Print Helm Templates
        run: helm template kubernetes/

      - name: Deploy Helm
        run: helm install exareme2 kubernetes/ --debug

      - name: Wait for pods to get healthy
        run: timeout 300 bash -c 'while true; do if kubectl get pods --no-headers | awk '\''{if ($2 != "1/1" && $2 != "2/2" && $2 != "3/3" && $2 != "4/4") exit 1;}'\''; then echo "All pods are ready!"; break; else kubectl get pods -o wide; sleep 20; fi done'

      - name: Load data models into localworkers and globalworker
        run: |
          LOCALWORKER1=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker1") | .metadata.name')
          LOCALWORKER2=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker2") | .metadata.name')
          GLOBALWORKER=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r ".items[0].metadata.name")

          for POD in $LOCALWORKER1 $LOCALWORKER2 $GLOBALWORKER; do
            kubectl exec $POD -c db-importer -- sh -c 'mipdb init'
            for model in dementia_v_0_1 tbi_v_0_1 longitudinal_dementia_v_0_1; do
              kubectl exec $POD -c db-importer -- sh -c "mipdb add-data-model /opt/data/${model}/CDEsMetadata.json"
            done
          done

      - name: Load Dataset CSVs into Localworkers and Globalworker
        run: |
          LOCALWORKER1=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker1") | .metadata.name')
          LOCALWORKER2=$(kubectl get pods -o json | jq -r '.items[] | select(.spec.nodeName=="localworker2") | .metadata.name')
          GLOBALWORKER=$(kubectl get pods -l=nodeType=globalworker -o json | jq -r ".items[0].metadata.name")

          for model in dementia_v_0_1 tbi_v_0_1 longitudinal_dementia_v_0_1; do
            for filepath in $(kubectl exec $GLOBALWORKER -c db-importer -- ls /opt/data/${model}); do
              filepath=/opt/data/${model}/${filepath}
              if [[ $filepath == *test.csv ]]; then
                echo "Loading file: $filepath at $GLOBALWORKER"
                kubectl exec $GLOBALWORKER -c db-importer -- mipdb add-dataset $filepath -d ${model%_v_*} -v 0.1
              elif [[ $filepath == *.csv ]]; then
                filename=$(basename $filepath)
                suffix=$(echo $filename | grep -o '[0-9]*' | tail -1)
                if (( suffix % 2 == 0 )); then
                  POD_NAME=$LOCALWORKER2
                else
                  POD_NAME=$LOCALWORKER1
                fi
                echo "Loading file: $filepath at $POD_NAME"
                kubectl exec $POD_NAME -c db-importer -- mipdb add-dataset $filepath -d ${model%_v_*} -v 0.1
              fi
            done
          done


      - name: Controller logs
        run: kubectl logs -l app=exareme2-controller --tail -1

      - name: Globalnode logs
        run: kubectl logs -l nodeType=globalworker -c worker --tail -1

      - name: Localnode logs
        run: kubectl logs -l nodeType=localworker -c worker --tail -1

      - name: Controller logs (post run)
        uses: webiny/action-post-run@3.0.0
        with:
          run: kubectl logs -l app=exareme2-controller --tail -1

      - name: Globalnode logs (post run)
        uses: webiny/action-post-run@3.0.0
        with:
          run: kubectl logs -l nodeType=globalworker -c worker --tail -1

      - name: Localnode logs (post run)
        uses: webiny/action-post-run@3.0.0
        with:
          run: kubectl logs -l nodeType=localworker -c worker --tail -1

      - name: Run Worker Landscape Aggregator update
        run: curl -X POST "http://172.17.0.1:5000/wla"

      - name: Run Healthcheck
        run: curl "http://172.17.0.1:5000/healthcheck"

      - name: Update WorkerLandscapeAggregator
        run: curl -X POST "http://172.17.0.1:5000/wla"

      - name: Run production env tests
        run: poetry run pytest tests/prod_env_tests --verbosity=4