tests: add vllm

Signed-off-by: Ettore Di Giacinto <[email protected]>

.github/workflows/test-extra.yml

@@ -132,4 +132,36 @@ jobs:
            make -C backend/python/transformers-musicgen test
 
 
+
+
+  tests-vllm:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+        with: 
+          submodules: true
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install build-essential ffmpeg
+          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
+             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
+              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
+             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
+             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
+             sudo apt-get update && \
+             sudo apt-get install -y conda
+          sudo apt-get install -y ca-certificates cmake curl patch
+          sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
+          
+          sudo rm -rfv /usr/bin/conda || true
+
+      - name: Test vllm
+        run: |
+           export PATH=$PATH:/opt/conda/bin
+           make -C backend/python/vllm
+           make -C backend/python/vllm test
+
+
  

backend/python/transformers-musicgen/Makefile

@@ -17,7 +17,6 @@ run:
 	bash run.sh
 	@echo "transformers run."
 
-# It is not working well by using command line. It only6 works with IDE like VSCode.
 .PHONY: test
 test:
 	@echo "Testing transformers..."

backend/python/vllm/Makefile

@@ -9,3 +9,9 @@ run:
 	@echo "Running vllm..."
 	bash run.sh
 	@echo "vllm run."
+
+.PHONY: test
+test:
+	@echo "Testing vllm..."
+	bash test.sh
+	@echo "vllm tested."

backend/python/vllm/test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+##
+## A bash script wrapper that runs the transformers server with conda
+
+# Activate conda environment
+source activate vllm
+
+# get the directory where the bash script is located
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+python -m unittest $DIR/test_backend_vllm.py

backend/python/vllm/test_backend_vllm.py

@@ -21,13 +21,13 @@ class TestBackendServicer(unittest.TestCase):
     """
     def setUp(self):
         self.service = subprocess.Popen(["python", "backend_vllm.py", "--addr", "localhost:50051"])
+        time.sleep(10)
 
     def tearDown(self) -> None:
         self.service.terminate()
         self.service.wait()
 
     def test_server_startup(self):
-        time.sleep(2)
         try:
             self.setUp()
             with grpc.insecure_channel("localhost:50051") as channel:
@@ -39,3 +39,38 @@ def test_server_startup(self):
             self.fail("Server failed to start")
         finally:
             self.tearDown()
+    def test_load_model(self):
+        """
+        This method tests if the model is loaded successfully
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
+                self.assertTrue(response.success)
+                self.assertEqual(response.message, "Model loaded successfully")
+        except Exception as err:
+            print(err)
+            self.fail("LoadModel service failed")
+        finally:
+            self.tearDown()
+
+    def test_text(self):
+        """
+        This method tests if the embeddings are generated successfully
+        """
+        try:
+            self.setUp()
+            with grpc.insecure_channel("localhost:50051") as channel:
+                stub = backend_pb2_grpc.BackendStub(channel)
+                response = stub.LoadModel(backend_pb2.ModelOptions(Model="facebook/opt-125m"))
+                self.assertTrue(response.success)
+                req = backend_pb2.PredictOptions(prompt="The capital of France is")
+                resp = stub.Predict(req)
+                self.assertIsNotNone(resp.message)
+        except Exception as err:
+            print(err)
+            self.fail("text service failed")
+        finally:
+            self.tearDown()