Mutation testing (#64)

* Improve tests by making them immune to mutations * Introduce mutations testing pipeline * Add mutation badge
Accenture · Sep 30, 2023 · fb27821 · fb27821
1 parent 144dbd1
commit fb27821
Show file tree

Hide file tree

Showing 18 changed files with 267 additions and 53 deletions.
diff --git a/.github/workflows/test-mutations.yml b/.github/workflows/test-mutations.yml
@@ -0,0 +1,56 @@
+name: test-mutations
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - "master" # Run the workflow when pushing to the master branch
+    paths-ignore:
+      - "**.md"
+      - "**.png"
+  pull_request:
+    branches:
+      - "*" # Run the workflow for all pull requests
+    paths-ignore:
+      - "**.md"
+      - "**.png"
+
+env:
+  DOTNET_NOLOGO: true
+  DOTNET_CLI_TELEMETRY_OPTOUT: true
+  Configuration: Release
+  StrongNameKeyName: key.snk
+
+defaults:
+  run:
+    shell: pwsh
+
+jobs:
+  init:
+    runs-on: ubuntu-latest
+    steps:
+      - name: 📥 checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: 🖊️ materialize signing key
+        id: signing-key
+        run: |
+          $path = [IO.Path]::Combine("${{ github.workspace }}","src","${{ env.StrongNameKeyName }}")
+          [IO.File]::WriteAllBytes($path, [Convert]::FromBase64String("$env:StrongNameKey"))
+          "PATH=$path" >> $env:GITHUB_OUTPUT
+        env:
+          StrongNameKey: ${{ secrets.SIGNING_KEY }}
+      - name: 💾 install stryker.net
+        run: |
+          dotnet new tool-manifest
+          dotnet tool install --local dotnet-stryker
+      - name: 👾 test mutations
+        working-directory: ./src
+        run: |
+          dotnet tool run dotnet-stryker -f stryker-config.yaml -r dashboard -v ${{ github.ref_name }} --dashboard-api-key ${{ secrets.STRYKER_API_KEY }} -V debug
+        env:
+          CI: true
+          StrongNameKey: ${{ secrets.SIGNING_KEY }}
+          StrongNameKeyPath: ${{ steps.signing-key.outputs.PATH }}
+
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Objectivity.AutoFixture.XUnit2.AutoMock
 
-[![CI/CD](https://github.com/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/actions/workflows/cicd.yml/badge.svg?branch=master)](https://github.com/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/actions/workflows/cicd.yml) [![codecov](https://codecov.io/gh/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/branch/master/graph/badge.svg)](https://codecov.io/gh/ObjectivityLtd/AutoFixture.XUnit2.AutoMock) [![License: MIT](https://img.shields.io/github/license/ObjectivityLtd/AutoFixture.XUnit2.AutoMock?label=License&color=brightgreen)](https://opensource.org/licenses/MIT) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FObjectivityLtd%2FAutoFixture.XUnit2.AutoMock.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2FObjectivityLtd%2FAutoFixture.XUnit2.AutoMock?ref=badge_shield)
+[![CI/CD](https://github.com/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/actions/workflows/cicd.yml/badge.svg?branch=master)](https://github.com/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/actions/workflows/cicd.yml) [![codecov](https://codecov.io/gh/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/branch/master/graph/badge.svg)](https://codecov.io/gh/ObjectivityLtd/AutoFixture.XUnit2.AutoMock) [![Mutation testing badge](https://img.shields.io/endpoint?style=flat&url=https%3A%2F%2Fbadge-api.stryker-mutator.io%2Fgithub.com%2FObjectivityLtd%2FAutoFixture.XUnit2.AutoMock%2Fmaster)](https://dashboard.stryker-mutator.io/reports/github.com/ObjectivityLtd/AutoFixture.XUnit2.AutoMock/master) [![License: MIT](https://img.shields.io/github/license/ObjectivityLtd/AutoFixture.XUnit2.AutoMock?label=License&color=brightgreen)](https://opensource.org/licenses/MIT) [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FObjectivityLtd%2FAutoFixture.XUnit2.AutoMock.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2FObjectivityLtd%2FAutoFixture.XUnit2.AutoMock?ref=badge_shield)
 
 Accelerates preparation of mocked structures for unit tests under  [XUnit2](http://xunit.github.io/) by configuring [AutoFixture](https://github.com/AutoFixture/AutoFixture) data generation to use a mocking library of your choice. Gracefully handles recursive structures by omitting recursions.
 

diff --git a/...ectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests/Attributes/AutoMockDataAttributeTests.cs b/...ectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests/Attributes/AutoMockDataAttributeTests.cs
@@ -1,6 +1,5 @@
 namespace Objectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests.Attributes
 {
-    using System;
     using System.Collections.Generic;
     using System.Reflection;
 
@@ -80,16 +79,24 @@ public void WhenGetDataIsInvoked_ThenFixtureIsConfiguredAndDataReturned(bool ign
         }
 
         [AutoMockData]
-        [Theory(DisplayName = "GIVEN test method has some parameters WHEN test run THEN parameters are generated")]
-        public void GivenTestMethodHasSomeParameters_WhenTestRun_ThenParametersAreGenerated(int value, IDisposable disposable)
+        [Theory(DisplayName = "GIVEN test method has some value parameters WHEN test run THEN parameters are generated")]
+        public void GivenTestMethodHasSomeValueParameters_WhenTestRun_ThenParametersAreGenerated(int value)
         {
             // Arrange
             // Act
             // Assert
             value.Should().NotBe(default);
+        }
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().Contain("Proxy", "that way we know it was mocked.");
+        [AutoMockData]
+        [Theory(DisplayName = "GIVEN test method has some object parameters WHEN test run THEN parameters are generated")]
+        public void GivenTestMethodHasSomeObjectParameters_WhenTestRun_ThenParametersAreGenerated(IFakeObjectUnderTest value)
+        {
+            // Arrange
+            // Act
+            // Assert
+            value.Should().NotBeNull();
+            value.StringProperty.Should().NotBeNullOrEmpty();
         }
 
         protected void MethodUnderTest()

diff --git a/...ty.AutoFixture.XUnit2.AutoFakeItEasy.Tests/Attributes/InlineAutoMockDataAttributeTests.cs b/...ty.AutoFixture.XUnit2.AutoFakeItEasy.Tests/Attributes/InlineAutoMockDataAttributeTests.cs
@@ -1,6 +1,5 @@
 namespace Objectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests.Attributes
 {
-    using System;
     using System.Collections.Generic;
     using System.Reflection;
 
@@ -114,15 +113,19 @@ public void WhenGetDataIsInvoked_ThenFixtureIsConfiguredAndDataReturned(bool ign
 
         [InlineAutoMockData(100)]
         [Theory(DisplayName = "GIVEN test method has some inline parameters WHEN test run THEN parameters are generated")]
-        public void GivenTestMethodHasSomeInlineParameters_WhenTestRun_ThenParametersAreGenerated(int value, IDisposable disposable)
+        public void GivenTestMethodHasSomeInlineParameters_WhenTestRun_ThenParametersAreGenerated(
+            int firstValueInstance,
+            int secondValueInstance,
+            IFakeObjectUnderTest objectInstance)
         {
             // Arrange
             // Act
             // Assert
-            value.Should().Be(100);
+            firstValueInstance.Should().Be(100);
+            secondValueInstance.Should().NotBe(default);
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().Contain("Proxy", "that way we know it was mocked.");
+            objectInstance.Should().NotBeNull();
+            objectInstance.StringProperty.Should().NotBeNullOrEmpty();
         }
 
         protected void MethodUnderTest()

diff --git a/...ty.AutoFixture.XUnit2.AutoFakeItEasy.Tests/Attributes/MemberAutoMockDataAttributeTests.cs b/...ty.AutoFixture.XUnit2.AutoFakeItEasy.Tests/Attributes/MemberAutoMockDataAttributeTests.cs
@@ -164,7 +164,12 @@ public void GivenUninitializedMemberName_WhenConstructorIsInvoked_ThenExceptionI
 
         [Theory(DisplayName = "GIVEN test method has some member generated parameters WHEN test run THEN parameters are provided")]
         [MemberAutoMockData(nameof(TestData))]
-        public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenParametersAreProvided(int first, int second, int third, int fourth, IDisposable disposable)
+        public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenParametersAreProvided(
+            int first,
+            int second,
+            int third,
+            int fourth,
+            IFakeObjectUnderTest objectInstance)
         {
             // Arrange
             var testData = TestData.ToList();
@@ -176,8 +181,8 @@ public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenPara
             third.Should().BeOneOf((int)testData[0][2], (int)testData[1][2], (int)testData[2][2]);
             fourth.Should().NotBe(default);
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().Contain("Proxy", "that way we know it was mocked.");
+            objectInstance.Should().NotBeNull();
+            objectInstance.StringProperty.Should().NotBeNullOrEmpty();
         }
     }
 }
diff --git a/src/Objectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests/IFakeObjectUnderTest.cs b/src/Objectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests/IFakeObjectUnderTest.cs
@@ -0,0 +1,7 @@
+namespace Objectivity.AutoFixture.XUnit2.AutoFakeItEasy.Tests
+{
+    public interface IFakeObjectUnderTest
+    {
+        public string StringProperty { get; set; }
+    }
+}
diff --git a/src/Objectivity.AutoFixture.XUnit2.AutoMoq.Tests/Attributes/AutoMockDataAttributeTests.cs b/src/Objectivity.AutoFixture.XUnit2.AutoMoq.Tests/Attributes/AutoMockDataAttributeTests.cs
@@ -1,6 +1,5 @@
 namespace Objectivity.AutoFixture.XUnit2.AutoMoq.Tests.Attributes
 {
-    using System;
     using System.Collections.Generic;
     using System.Reflection;
 
@@ -80,16 +79,24 @@ public void WhenGetDataIsInvoked_ThenFixtureIsConfiguredAndDataReturned(bool ign
         }
 
         [AutoMockData]
-        [Theory(DisplayName = "GIVEN test method has some parameters WHEN test run THEN parameters are generated")]
-        public void GivenTestMethodHasSomeParameters_WhenTestRun_ThenParametersAreGenerated(int value, IDisposable disposable)
+        [Theory(DisplayName = "GIVEN test method has some value parameters WHEN test run THEN parameters are generated")]
+        public void GivenTestMethodHasSomeValueParameters_WhenTestRun_ThenParametersAreGenerated(int value)
         {
             // Arrange
             // Act
             // Assert
             value.Should().NotBe(default);
+        }
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().StartWith("IDisposableProxy", "that way we know it was mocked.");
+        [AutoMockData]
+        [Theory(DisplayName = "GIVEN test method has some object parameters WHEN test run THEN parameters are generated")]
+        public void GivenTestMethodHasSomeObjectParameters_WhenTestRun_ThenParametersAreGenerated(IFakeObjectUnderTest value)
+        {
+            // Arrange
+            // Act
+            // Assert
+            value.Should().NotBeNull();
+            value.StringProperty.Should().NotBeNullOrEmpty();
         }
 
         protected void MethodUnderTest()

diff --git a/...jectivity.AutoFixture.XUnit2.AutoMoq.Tests/Attributes/InlineAutoMockDataAttributeTests.cs b/...jectivity.AutoFixture.XUnit2.AutoMoq.Tests/Attributes/InlineAutoMockDataAttributeTests.cs
@@ -1,6 +1,5 @@
 namespace Objectivity.AutoFixture.XUnit2.AutoMoq.Tests.Attributes
 {
-    using System;
     using System.Collections.Generic;
     using System.Reflection;
 
@@ -114,15 +113,19 @@ public void WhenGetDataIsInvoked_ThenFixtureIsConfiguredAndDataReturned(bool ign
 
         [InlineAutoMockData(100)]
         [Theory(DisplayName = "GIVEN test method has some inline parameters WHEN test run THEN parameters are generated")]
-        public void GivenTestMethodHasSomeInlineParameters_WhenTestRun_ThenParametersAreGenerated(int value, IDisposable disposable)
+        public void GivenTestMethodHasSomeInlineParameters_WhenTestRun_ThenParametersAreGenerated(
+            int firstValueInstance,
+            int secondValueInstance,
+            IFakeObjectUnderTest objectInstance)
         {
             // Arrange
             // Act
             // Assert
-            value.Should().Be(100);
+            firstValueInstance.Should().Be(100);
+            secondValueInstance.Should().NotBe(default);
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().StartWith("IDisposableProxy", "that way we know it was mocked.");
+            objectInstance.Should().NotBeNull();
+            objectInstance.StringProperty.Should().NotBeNullOrEmpty();
         }
 
         protected void MethodUnderTest()

diff --git a/...jectivity.AutoFixture.XUnit2.AutoMoq.Tests/Attributes/MemberAutoMockDataAttributeTests.cs b/...jectivity.AutoFixture.XUnit2.AutoMoq.Tests/Attributes/MemberAutoMockDataAttributeTests.cs
@@ -164,7 +164,12 @@ public void GivenUninitializedMemberName_WhenConstructorIsInvoked_ThenExceptionI
 
         [Theory(DisplayName = "GIVEN test method has some member generated parameters WHEN test run THEN parameters are provided")]
         [MemberAutoMockData(nameof(TestData))]
-        public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenParametersAreProvided(int first, int second, int third, int fourth, IDisposable disposable)
+        public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenParametersAreProvided(
+            int first,
+            int second,
+            int third,
+            int fourth,
+            IFakeObjectUnderTest objectInstance)
         {
             // Arrange
             var testData = TestData.ToList();
@@ -176,8 +181,8 @@ public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenPara
             third.Should().BeOneOf((int)testData[0][2], (int)testData[1][2], (int)testData[2][2]);
             fourth.Should().NotBe(default);
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().StartWith("IDisposableProxy", "that way we know it was mocked.");
+            objectInstance.Should().NotBeNull();
+            objectInstance.StringProperty.Should().NotBeNullOrEmpty();
         }
     }
 }
diff --git a/src/Objectivity.AutoFixture.XUnit2.AutoMoq.Tests/IFakeObjectUnderTest.cs b/src/Objectivity.AutoFixture.XUnit2.AutoMoq.Tests/IFakeObjectUnderTest.cs
@@ -0,0 +1,7 @@
+namespace Objectivity.AutoFixture.XUnit2.AutoMoq.Tests
+{
+    public interface IFakeObjectUnderTest
+    {
+        public string StringProperty { get; set; }
+    }
+}
diff --git a/...ctivity.AutoFixture.XUnit2.AutoNSubstitute.Tests/Attributes/AutoMockDataAttributeTests.cs b/...ctivity.AutoFixture.XUnit2.AutoNSubstitute.Tests/Attributes/AutoMockDataAttributeTests.cs
@@ -1,6 +1,5 @@
 namespace Objectivity.AutoFixture.XUnit2.AutoNSubstitute.Tests.Attributes
 {
-    using System;
     using System.Collections.Generic;
     using System.Reflection;
 
@@ -79,16 +78,24 @@ public void WhenGetDataIsInvoked_ThenFixtureIsConfiguredAndDataReturned(bool ign
         }
 
         [AutoMockData]
-        [Theory(DisplayName = "GIVEN test method has some parameters WHEN test run THEN parameters are generated")]
-        public void GivenTestMethodHasSomeParameters_WhenTestRun_ThenParametersAreGenerated(int value, IDisposable disposable)
+        [Theory(DisplayName = "GIVEN test method has some value parameters WHEN test run THEN parameters are generated")]
+        public void GivenTestMethodHasSomeValueParameters_WhenTestRun_ThenParametersAreGenerated(int value)
         {
             // Arrange
             // Act
             // Assert
             value.Should().NotBe(default);
+        }
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().StartWith("ObjectProxy", "that way we know it was mocked.");
+        [AutoMockData]
+        [Theory(DisplayName = "GIVEN test method has some object parameters WHEN test run THEN parameters are generated")]
+        public void GivenTestMethodHasSomeObjectParameters_WhenTestRun_ThenParametersAreGenerated(IFakeObjectUnderTest value)
+        {
+            // Arrange
+            // Act
+            // Assert
+            value.Should().NotBeNull();
+            value.StringProperty.Should().NotBeNullOrEmpty();
         }
 
         protected void MethodUnderTest()

diff --git a/...y.AutoFixture.XUnit2.AutoNSubstitute.Tests/Attributes/InlineAutoMockDataAttributeTests.cs b/...y.AutoFixture.XUnit2.AutoNSubstitute.Tests/Attributes/InlineAutoMockDataAttributeTests.cs
@@ -1,6 +1,5 @@
 namespace Objectivity.AutoFixture.XUnit2.AutoNSubstitute.Tests.Attributes
 {
-    using System;
     using System.Collections.Generic;
     using System.Reflection;
 
@@ -113,15 +112,19 @@ public void WhenGetDataIsInvoked_ThenFixtureIsConfiguredAndDataReturned(bool ign
 
         [InlineAutoMockData(100)]
         [Theory(DisplayName = "GIVEN test method has some inline parameters WHEN test run THEN parameters are generated")]
-        public void GivenTestMethodHasSomeInlineParameters_WhenTestRun_ThenParametersAreGenerated(int value, IDisposable disposable)
+        public void GivenTestMethodHasSomeInlineParameters_WhenTestRun_ThenParametersAreGenerated(
+            int firstValueInstance,
+            int secondValueInstance,
+            IFakeObjectUnderTest objectInstance)
         {
             // Arrange
             // Act
             // Assert
-            value.Should().Be(100);
+            firstValueInstance.Should().Be(100);
+            secondValueInstance.Should().NotBe(default);
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().StartWith("ObjectProxy", "that way we know it was mocked.");
+            objectInstance.Should().NotBeNull();
+            objectInstance.StringProperty.Should().NotBeNullOrEmpty();
         }
 
         protected void MethodUnderTest()

diff --git a/...y.AutoFixture.XUnit2.AutoNSubstitute.Tests/Attributes/MemberAutoMockDataAttributeTests.cs b/...y.AutoFixture.XUnit2.AutoNSubstitute.Tests/Attributes/MemberAutoMockDataAttributeTests.cs
@@ -163,7 +163,12 @@ public void GivenUninitializedMemberName_WhenConstructorIsInvoked_ThenExceptionI
 
         [Theory(DisplayName = "GIVEN test method has some member generated parameters WHEN test run THEN parameters are provided")]
         [MemberAutoMockData(nameof(TestData))]
-        public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenParametersAreProvided(int first, int second, int third, int fourth, IDisposable disposable)
+        public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenParametersAreProvided(
+            int first,
+            int second,
+            int third,
+            int fourth,
+            IFakeObjectUnderTest objectInstance)
         {
             // Arrange
             var testData = TestData.ToList();
@@ -175,8 +180,8 @@ public void GivenTestMethodHasSomeMemberGeneratedParameters_WhenTestRun_ThenPara
             third.Should().BeOneOf((int)testData[0][2], (int)testData[1][2], (int)testData[2][2]);
             fourth.Should().NotBe(default);
 
-            disposable.Should().NotBeNull();
-            disposable.GetType().Name.Should().StartWith("ObjectProxy", "that way we know it was mocked.");
+            objectInstance.Should().NotBeNull();
+            objectInstance.StringProperty.Should().NotBeNullOrEmpty();
         }
     }
 }
diff --git a/src/Objectivity.AutoFixture.XUnit2.AutoNSubstitute.Tests/IFakeObjectUnderTest.cs b/src/Objectivity.AutoFixture.XUnit2.AutoNSubstitute.Tests/IFakeObjectUnderTest.cs
@@ -0,0 +1,7 @@
+namespace Objectivity.AutoFixture.XUnit2.AutoNSubstitute.Tests
+{
+    public interface IFakeObjectUnderTest
+    {
+        public string StringProperty { get; set; }
+    }
+}