Merge pull request #19 from NOEL-MNI/github-actions_self-hosted

inference pipeline compatible with self-hosted runner for CI

.github/workflows/test-inference-pipeline.yml

@@ -0,0 +1,48 @@
+
+name: Testing Inference Pipeline
+
+on: [push, pull_request]
+
+env:
+  CI_TESTING: True
+
+jobs:
+  build:
+
+    runs-on: self-hosted
+
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install dependencies for deepMask
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda create -n preprocess python=3.8
+          conda activate preprocess
+          python -m pip install -r ./app/deepMask/app/requirements.txt
+          conda deactivate
+
+      - name: Install dependencies for deepFCD
+        run: |
+          python -m pip install -r ./app/requirements.txt
+          conda install -c conda-forge pygpu==0.7.6
+          pip cache purge
+
+      - name: Download openneuro.org dataset to test the inference pipeline # https://openneuro.org/datasets/ds004199/versions/1.0.5
+        run: |
+          PATIENT_ID=sub-00055
+          BASE_URL=https://s3.amazonaws.com/openneuro.org/ds004199/${PATIENT_ID}/anat
+          mkdir -p ~/io/${PATIENT_ID}
+          echo "retrieving single-patient multimodal dataset.."
+          wget ${BASE_URL}/${PATIENT_ID}_acq-sag111_T1w.nii.gz\?versionId\=IKGWDiLR7B7ls2yPVyycJo.6R1Sqhujf -O ~/io/sub-00055/t1.nii.gz
+          wget ${BASE_URL}/${PATIENT_ID}_acq-tse3dvfl_FLAIR.nii.gz\?versionId\=HmzYoUuYkdbyd8jkpdJjVkZydRHNSqUX -O ~/io/sub-00055/flair.nii.gz
+          wget ${BASE_URL}/${PATIENT_ID}_acq-tse3dvfl_FLAIR_roi.nii.gz\?versionId\=ulmEU3nb8WCvGwcwTbkcdNSVr07PMPQN -O ~/io/sub-00055/label.nii.gz
+
+      - name: Run inference for deepFCD
+        run: |
+          ./app/inference.py sub-00055 t1.nii.gz flair.nii.gz ~/io cuda 1 1
+        env:
+          CI_TESTING_GT: "./tests/segmentations/sub-00055/sub-00055_label_dilated_final.nii.gz"

.gitignore

@@ -4,4 +4,5 @@ notebooks/
 .ipynb_checkpoints/
 *.csv
 memray*.bin
-memray*.html
+memray*.html
+.env

Makefile

@@ -1,10 +1,10 @@
 ACCOUNT := noelmni
 SERVICE := deep-fcd
 IMAGE 	:= $(ACCOUNT)/$(SERVICE) # noelmni/deep-fcd
-TAG		:= test
+TAG		:= resamp_orig
 UID		:= 2551
 GID		:= 618
-CASE_ID := BAR_002
+CASE_ID := sub-00055
 TMPDIR	:= /host/hamlet/local_raid/data/ravnoor/sandbox
 BRAIN_MASKING := 1
 PREPROCESS		:= 1
@@ -35,7 +35,17 @@ test-pipeline-docker:
 	--user="$(UID):$(GID)" \
 	--volume="$(TMPDIR):/tmp" \
 	$(ACCOUNT)/$(SERVICE):$(TAG) \
-	/app/inference.py $(CASE_ID) t1.nii.gz flair.nii.gz /tmp cuda0 $(BRAIN_MASKING) $(PREPROCESS)
+	/app/inference.py $(CASE_ID) T1.nii.gz FLAIR.nii.gz /tmp cuda0 $(BRAIN_MASKING) $(PREPROCESS)
+
+test-pipeline-docker_ci:
+	docker run --rm -it --init \
+	--gpus=all	\
+	--user="$(UID):$(GID)" \
+	--volume="$(TMPDIR):/tmp" \
+	--env CI_TESTING=1 \
+	--env CI_TESTING_GT=/tmp/$(CASE_ID)/label_final_MD.nii.gz \
+	$(ACCOUNT)/$(SERVICE):$(TAG) \
+	/app/inference.py $(CASE_ID) T1.nii.gz FLAIR.nii.gz /tmp cuda0 $(BRAIN_MASKING) $(PREPROCESS)
 
 test-reporting:
 	./app/utils/reporting.py $(CASE_ID) $(TMPDIR)/
@@ -55,4 +65,25 @@ docker-clean:
 	rm -f /tmp/$(CASE_ID)/{*_final,*denseCrf3d*,*_native,*_maskpred}.nii.gz
 
 prune:
-	docker image prune
+	docker image prune
+
+runner-build:
+	docker-compose -f runner.docker-compose.yml build
+
+runner-ps:
+	docker-compose -f runner.docker-compose.yml ps
+
+runner-up:
+	docker-compose -f runner.docker-compose.yml up --remove-orphans -d
+
+runner-down:
+	docker-compose -f runner.docker-compose.yml down
+
+runner-logs:
+	docker-compose -f runner.docker-compose.yml logs -f
+
+runner-scale:
+	docker-compose up --scale runner=1 -d
+
+runner-bash:
+	docker-compose -f runner.docker-compose.yml exec -it runner bash

README.md

@@ -188,7 +188,7 @@ chmod +x ./app/utils/reporting.py
 ```
 #### example:
 ```bash
-./app/utils/inference.py FCD_001 /io
+./app/utils/reporting.py FCD_001 /io
 ```
 
 ### 4.2 Reporting output using Docker

app/inference.py

@@ -109,6 +109,14 @@
         "Skipping image preprocessing and brain masking, presumably images are co-registered, bias-corrected, and skull-stripped"
     )
 
+if os.environ.get("CI_TESTING") is not None:
+    options["CI_TESTING_GT"] = os.environ.get("CI_TESTING_GT")
+    print("CI environment initialized: {}".format(options["CI_TESTING_GT"]))
+    mask = ants.image_read(options["CI_TESTING_GT"])
+    t1, t2 = ants.image_read(args.t1), ants.image_read(args.t2)
+    ants.mask_image(t1, mask, level=1, binarize=False).to_filename(args.t1)
+    ants.mask_image(t2, mask, level=1, binarize=False).to_filename(args.t2)
+
 # deepFCD configuration
 K.set_image_dim_ordering("th")
 K.set_image_data_format("channels_first")  # TH dimension ordering in this code

runner.Dockerfile

@@ -0,0 +1,67 @@
+FROM noelmni/cuda:10.0-cudnn7-devel-ubuntu18.04
+LABEL maintainer="Ravnoor Singh Gill <[email protected]>" \
+        org.opencontainers.image.title="Self-hosted Github Actions runner for deepFCD" \
+        org.opencontainers.image.description="Automated Detection of Focal Cortical Dysplasia using Deep Learning" \
+        org.opencontainers.image.licenses="BSD-3-Clause" \
+        org.opencontainers.image.source="https://github.com/NOEL-MNI/deepFCD" \
+        org.opencontainers.image.url="https://github.com/NOEL-MNI/deepFCD"
+
+# manually update outdated repository key
+# fixes invalid GPG error: https://forums.developer.nvidia.com/t/gpg-error-http-developer-download-nvidia-com-compute-cuda-repos-ubuntu1804-x86-64/212904
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
+
+ARG RUNNER_VERSION=2.309.0
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update -y
+RUN apt-get install -y --no-install-recommends \
+    bash build-essential curl jq libssl-dev libffi-dev \
+    nano python3-dev software-properties-common unzip wget
+
+# install git 2.17+
+RUN add-apt-repository ppa:git-core/candidate -y
+RUN apt-get update
+RUN apt-get install -y git
+
+RUN apt-get remove nodejs npm
+
+# github actions needs a non-root to run
+RUN useradd -m ga 
+WORKDIR /home/ga/actions-runner
+ENV HOME=/home/ga
+
+# https://stackoverflow.com/questions/25899912/how-to-install-nvm-in-docker/60137919#60137919
+SHELL ["/bin/bash", "--login", "-i", "-c"]
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.5/install.sh | bash
+RUN source /root/.bashrc && nvm install 16
+SHELL ["/bin/bash", "--login", "-c"]
+
+# install Github Actions runner
+RUN curl -s -O -L https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    tar xzf ./actions-runner-linux-x64-${RUNNER_VERSION}.tar.gz && \
+    ./bin/installdependencies.sh
+
+# RUN wget -c https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz && \
+#     ./bin/patchelf --set-interpreter /opt/glibc-2.28/lib/ld-linux-x86-64.so.2 --set-rpath /opt/glibc-2.28/lib/ /home/ga/.nvm/versions/node/v20.6.1/bin/node
+
+# add over the start.sh script
+ADD start-runner.sh start.sh
+
+# make the script executable
+RUN chmod +x start.sh
+
+# set permission and user to ga
+RUN chown -R ga /home/ga
+USER ga
+
+# install Conda
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh && \
+    /bin/bash Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -b && \
+    rm Miniconda3-py38_23.5.2-0-Linux-x86_64.sh && \
+    echo '. ~/miniconda3/etc/profile.d/conda.sh' >> ~/.bashrc
+
+# create a dir to store inputs and outputs
+RUN mkdir ~/io
+
+# set the entrypoint to the start.sh script
+ENTRYPOINT ["./start.sh"]

runner.docker-compose.yml

@@ -0,0 +1,28 @@
+version: '3.9'
+
+services:
+  runner:
+    image: noelmni/deep-fcd:runner_alpha
+    # command: '/app/inference.py FCD_001 T1.nii.gz FLAIR.nii.gz /io cuda0 1 1'
+    # command: nvidia-smi
+    # entrypoint: /bin/bash
+    build:
+      context: .
+      dockerfile: runner.Dockerfile
+      args:
+        RUNNER_VERSION: '2.309.0'
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              # count: 1
+              device_ids: ['1']
+              capabilities: [gpu]
+    # volumes:
+        # - '$PWD/io:/io'
+        # - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      GH_TOKEN: ${GH_TOKEN}
+      GH_OWNER: ${GH_OWNER}
+      GH_REPOSITORY: ${GH_REPOSITORY}

start-runner.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+GH_OWNER=${GH_OWNER}
+GH_REPOSITORY=${GH_REPOSITORY}
+GH_TOKEN=${GH_TOKEN}
+
+HOSTNAME=$(hostname | cut -d "." -f 1)
+RUNNER_SUFFIX=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 7 | head -n 1)
+RUNNER_NAME="minion-${HOSTNAME}-${RUNNER_SUFFIX}"
+
+echo ${RUNNER_NAME}
+
+REG_TOKEN=$(curl -sX POST -H "Accept: application/vnd.github+json" -H "Authorization: token ${GH_TOKEN}" https://api.github.com/repos/${GH_OWNER}/${GH_REPOSITORY}/actions/runners/registration-token | jq .token --raw-output)
+
+cd /home/ga/actions-runner
+
+./config.sh --unattended --url https://github.com/${GH_OWNER}/${GH_REPOSITORY} --token ${REG_TOKEN} --name ${RUNNER_NAME}
+
+cleanup() {
+    echo "Removing runner..."
+    ./config.sh remove --unattended --token ${REG_TOKEN}
+}
+
+export PATH=/home/ga/miniconda3/condabin:/home/ga/miniconda3/bin:$PATH
+
+trap 'cleanup; exit 130' INT
+trap 'cleanup; exit 143' TERM
+
+./run.sh & wait $!