Merge pull request #1 from helixml/fix/usdgbp

improve currency pair responses
helixml · Nov 10, 2024 · a2b3909 · a2b3909
2 parents d121303 + f08a98a
commit a2b3909
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 4 deletions.
diff --git a/.github/workflows/helix.yml b/.github/workflows/helix.yml
@@ -20,10 +20,10 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install helix CLI
-        run: curl -sL -O https://get.helix.ml/install.sh && bash install.sh --cli -y --helix-version 1.4.0-rc11
+        run: curl -sL -O https://get.helix.ml/install.sh && bash install.sh --cli -y --helix-version 1.4.0-rc12
 
       - name: Test the helix app
-        run: helix test -f aispecs/exchangerates.yaml
+        run: helix test -f aispecs/exchangerates.yaml --evaluation-model meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
 
       - name: PR comment with file
         if: always()
@@ -38,7 +38,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Install helix CLI
-        run: curl -sL -O https://get.helix.ml/install.sh && bash install.sh --cli -y --helix-version 1.4.0-rc11
+        run: curl -sL -O https://get.helix.ml/install.sh && bash install.sh --cli -y --helix-version 1.4.0-rc12
 
       - name: Test the helix app
-        run: helix test -f aispecs/exchangerates.yaml
+        run: helix test -f aispecs/exchangerates.yaml --evaluation-model meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
diff --git a/aispecs/exchangerates.yaml b/aispecs/exchangerates.yaml
@@ -12,6 +12,10 @@ spec:
         steps:
           - prompt: what is the usd to gbp exchange rate?
             expected_output: the usd to gbp exchange rate
+      - name: usdgbp
+        steps:
+          - prompt: usdgbp
+            expected_output: the usd to gbp exchange rate only. it should specifically mention both usd and gbp and not other currencies. if it mentions other currencies, FAIL the test
       apis:
         - name: Exchange Rates API
           description: Get latest currency exchange rates