Enforce code coverage threshold (#368)

- New CI step (and Makefile local option) to check code coverage
- Enforce 90% line coverage for unit tests
- Run code coverage CI steps on Linux (since using Bash scripts)
- Only run code coverage check on net6.0 to avoid issues with missing dotnet versions on Linux machine

.github/workflows/ci.yml

@@ -25,9 +25,17 @@ jobs:
         run: make scan
         # TODO: In the future, we can collect the output logs by enabling Code Scanning and using the pre-built GitHub Action: https://github.com/marketplace/actions/securitycodescan
         # https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning/uploading-a-sarif-file-to-github#uploading-a-code-scanning-analysis-with-github-actions
-  coverage:
+  coverage_requirement:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up dotnet tools and dependencies
+        run: make install
+      - name: Check if test suite coverage meets requirements
+        run: make coverage-check
+  coverage_upload:
     if: github.ref == 'refs/heads/master'
-    runs-on: windows-2022
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
       - name: Set up dotnet tools and dependencies

EasyPost.Tests/EasyPost.Tests.csproj

@@ -31,6 +31,10 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="coverlet.msbuild" Version="3.2.0">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
     <PackageReference Include="EasyVCR" Version="0.5.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="[17.3.0, 18.0.0)" />
     <PackageReference Include="coverlet.collector" Version="[3.1.2, 4.0.0)">

EasyPost.Tests/packages.lock.json

@@ -8,6 +8,12 @@
         "resolved": "3.1.2",
         "contentHash": "wuLDIDKD5XMt0A7lE31JPenT7QQwZPFkP5rRpdJeblyXZ9MGLI8rYjvm5fvAKln+2/X+4IxxQDxBtwdrqKNLZw=="
       },
+      "coverlet.msbuild": {
+        "type": "Direct",
+        "requested": "[3.2.0, )",
+        "resolved": "3.2.0",
+        "contentHash": "lu/eJJpqJb4qy3BGPtDD/LI5RSOwXYYyRErTyaG0OTP69llzVK3FEe74hBQx0JtLUTLEVBfERV4uGYcE1Br2sg=="
+      },
       "EasyVCR": {
         "type": "Direct",
         "requested": "[0.5.1, )",
@@ -260,6 +266,12 @@
         "resolved": "3.1.2",
         "contentHash": "wuLDIDKD5XMt0A7lE31JPenT7QQwZPFkP5rRpdJeblyXZ9MGLI8rYjvm5fvAKln+2/X+4IxxQDxBtwdrqKNLZw=="
       },
+      "coverlet.msbuild": {
+        "type": "Direct",
+        "requested": "[3.2.0, )",
+        "resolved": "3.2.0",
+        "contentHash": "lu/eJJpqJb4qy3BGPtDD/LI5RSOwXYYyRErTyaG0OTP69llzVK3FEe74hBQx0JtLUTLEVBfERV4uGYcE1Br2sg=="
+      },
       "EasyVCR": {
         "type": "Direct",
         "requested": "[0.5.1, )",
@@ -550,6 +562,12 @@
         "resolved": "3.1.2",
         "contentHash": "wuLDIDKD5XMt0A7lE31JPenT7QQwZPFkP5rRpdJeblyXZ9MGLI8rYjvm5fvAKln+2/X+4IxxQDxBtwdrqKNLZw=="
       },
+      "coverlet.msbuild": {
+        "type": "Direct",
+        "requested": "[3.2.0, )",
+        "resolved": "3.2.0",
+        "contentHash": "lu/eJJpqJb4qy3BGPtDD/LI5RSOwXYYyRErTyaG0OTP69llzVK3FEe74hBQx0JtLUTLEVBfERV4uGYcE1Br2sg=="
+      },
       "EasyVCR": {
         "type": "Direct",
         "requested": "[0.5.1, )",
@@ -801,6 +819,12 @@
         "resolved": "3.1.2",
         "contentHash": "wuLDIDKD5XMt0A7lE31JPenT7QQwZPFkP5rRpdJeblyXZ9MGLI8rYjvm5fvAKln+2/X+4IxxQDxBtwdrqKNLZw=="
       },
+      "coverlet.msbuild": {
+        "type": "Direct",
+        "requested": "[3.2.0, )",
+        "resolved": "3.2.0",
+        "contentHash": "lu/eJJpqJb4qy3BGPtDD/LI5RSOwXYYyRErTyaG0OTP69llzVK3FEe74hBQx0JtLUTLEVBfERV4uGYcE1Br2sg=="
+      },
       "EasyVCR": {
         "type": "Direct",
         "requested": "[0.5.1, )",

Makefile

@@ -25,6 +25,10 @@ clean:
 coverage:
 	./generate_test_reports.sh
 
+## coverage-check - Check if the coverage is above the minimum threshold
+coverage-check:
+	./check_coverage.sh
+
 ## format - Formats the project
 format:
 	dotnet dotnet-format --no-restore
@@ -122,4 +126,4 @@ test-fw:
 uninstall-scanner:
 	dotnet tool uninstall security-scan
 
-.PHONY: help build build-test-fw build-prod clean coverage format install-cert install-tools install lint lint-scripts pre-release publish-all publish release restore scan setup-win setup-unix sign test test-fw uninstall-scanner
+.PHONY: help build build-test-fw build-prod clean coverage coverage-check format install-cert install-tools install lint lint-scripts pre-release publish-all publish release restore scan setup-win setup-unix sign test test-fw uninstall-scanner

check_coverage.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+THRESHOLD=90
+THRESHOLD_TYPE=line
+
+# Analyze test coverage
+test_folder="EasyPost.Tests"
+cd "$test_folder" || exit
+
+# Coverlet as a global tool does not report coverage properly (https://github.com/coverlet-coverage/coverlet/blob/2b8a4565b101ca70c3eaa9b5228c449dd3b81ad1/Documentation/GlobalTool.md)
+# Instead, use the coverlet.msbuild package
+# https://codeburst.io/code-coverage-in-net-core-projects-c3d6536fd7d7
+# This will fail to run on machines without MSBuild installed
+
+# If the coverage is below the threshold, exit with a non-zero exit code
+dotnet test /p:CollectCoverage=true /p:CoverletOutput=TestResults/ /p:Threshold=$THRESHOLD /p:ThresholdType=$THRESHOLD_TYPE /p:TargetFramework=net6.0

generate_test_reports.sh

@@ -15,6 +15,8 @@ done
 # Generate Cobertura report for each test project
 test_folder="EasyPost.Tests"
 cd "$test_folder" || exit
+
+# https://learn.microsoft.com/en-us/dotnet/core/tools/dotnet-test#:~:text=To%20collect%20code%20coverage%20on%20any%20platform%20that%20is%20supported%20by%20.NET%20Core%2C
 dotnet test --collect:"XPlat Code Coverage"
 
 # Generate HTML and lcov report