Add automated tests

Build.ps1

@@ -16,30 +16,6 @@ Process {
   $outputDir = Join-Path $PSScriptRoot ".output"
   $nupkgsPath = Join-Path $outputDir "*.nupkg"
 
-  $testProjectName = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetRandomFileName())
-  $testProjectDir = Join-Path $outputDir $testProjectName
-  $testProjectPath = Join-Path $testProjectDir "$testProjectName.csproj"
-  $testNugetConfigPath = Join-Path $testProjectDir "nuget.config"
-  $testNugetConfigContents = @"
-<?xml version="1.0" encoding="utf-8"?>
-<configuration>
-  <packageSources>
-    <clear />
-    <add key="nuget" value="https://api.nuget.org/v3/index.json" />
-    <add key="local" value="$outputDir" />
-  </packageSources>
-
-  <packageSourceMapping>
-    <packageSource key="nuget">
-      <package pattern="*" />
-    </packageSource>
-    <packageSource key="local">
-      <package pattern="Workleap.DotNet.CodingStandards" />
-    </packageSource>
-  </packageSourceMapping>
-</configuration>
-"@
-
   try {
     Push-Location $workingDir
     Remove-Item $outputDir -Force -Recurse -ErrorAction SilentlyContinue
@@ -54,14 +30,8 @@ Process {
     # Pack using NuGet.exe
     Exec { & nuget pack Workleap.DotNet.CodingStandards.nuspec -OutputDirectory $outputDir -Version $version -ForceEnglishOutput }
 
-    # Create a new test console project, add our newly created package and try to build it in release mode
-    # The default .NET console project template with top-level statements should not trigger any warnings
-    # We treat warnings as errors even though it's supposed to be already enabled by our package,
-    # just in case the package is not working as expected
-    Exec { & dotnet new console --name $testProjectName --output $testProjectDir }
-    Set-Content -Path $testNugetConfigPath -Value $testNugetConfigContents
-    Exec { & dotnet add $testProjectPath package Workleap.DotNet.CodingStandards --version $version }
-    Exec { & dotnet build $testProjectPath --configuration Release /p:TreatWarningsAsErrors=true }
+    # Run tests
+    Exec { & dotnet test }
 
     # Push to a NuGet feed if the environment variables are set
     if (($null -ne $env:NUGET_SOURCE ) -and ($null -ne $env:NUGET_API_KEY)) {

src/build/Workleap.DotNet.CodingStandards.props

@@ -0,0 +1,29 @@
+<Project>
+  <PropertyGroup>
+    <ReportAnalyzer Condition="'$(ReportAnalyzer)' == ''">true</ReportAnalyzer>
+    <Features Condition="'$(Features)' == ''">strict</Features>
+    <Deterministic Condition="'$(Deterministic)' == ''">true</Deterministic>
+    <EnableNETAnalyzers Condition="'$(EnableNETAnalyzers)' == ''">true</EnableNETAnalyzers>
+    <AnalysisLevel Condition="'$(AnalysisLevel)' == ''">latest-all</AnalysisLevel>
+    <EnforceCodeStyleInBuild Condition="'$(EnforceCodeStyleInBuild)' == ''">true</EnforceCodeStyleInBuild>
+
+    <!-- https://learn.microsoft.com/en-us/nuget/release-notes/nuget-5.5#summary-whats-new-in-55 -->
+    <RestoreUseStaticGraphEvaluation Condition="'$(RestoreUseStaticGraphEvaluation)' == ''">true</RestoreUseStaticGraphEvaluation>
+
+    <!-- Enable ContinuousIntegrationBuild when running on CI -->
+    <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">true</ContinuousIntegrationBuild>
+    <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">true</ContinuousIntegrationBuild>
+    <ContinuousIntegrationBuild Condition="'$(GITLAB_CI)' == 'true'">true</ContinuousIntegrationBuild>
+    <ContinuousIntegrationBuild Condition="'$(CI)' == 'true'">true</ContinuousIntegrationBuild>
+    <ContinuousIntegrationBuild Condition="'$(TEAMCITY_VERSION)' != ''">true</ContinuousIntegrationBuild>
+
+    <!-- TreatWarningsAsErrors is enabled for release builds, unless explicitly set -->
+    <TreatWarningsAsErrors Condition="'$(Configuration)' == 'Release' AND '$(TreatWarningsAsErrors)' == ''">true</TreatWarningsAsErrors>
+
+    <!-- https://devblogs.microsoft.com/visualstudio/vs-toolbox-accelerate-your-builds-of-sdk-style-net-projects/ -->
+    <AccelerateBuildsInVisualStudio Condition="'$(AccelerateBuildsInVisualStudio)' == ''">true</AccelerateBuildsInVisualStudio>
+
+    <!-- GenerateDocumentationFile must be set to true for IDE0005 (Remove unnecessary usings/imports) to work -->
+    <GenerateDocumentationFile Condition="'$(GenerateDocumentationFile)' == ''">true</GenerateDocumentationFile>
+  </PropertyGroup>
+</Project>

src/build/Workleap.DotNet.CodingStandards.targets

@@ -0,0 +1,31 @@
+<Project>
+  <!-- Register the EditorConfig files to the project -->
+  <!-- Remember that a particular .NET analysis rule can only be configured once across all imported global EditorConfig files -->
+  <ItemGroup>
+    <!-- Basic EditorConfig settings such as encoding, indentation, etc. -->
+    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\1_FileDefaults.editorconfig" />
+
+    <!-- C# code style -->
+    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\2_CodeStyle.editorconfig" />
+
+    <!-- .NET analyzers configuration for all projects, enforcing C# code style, quality, performance and security -->
+    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\3_AllProjectsAnalyzers.editorconfig" />
+
+    <!-- Configure ReSharper analyzers that overlaps with built-in .NET analyzers (only appears in Rider and VisualStudio IDEs) -->
+    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\4_ReSharperAnalyzers.editorconfig" />
+
+    <!-- .NET analyzers configuration only for test projects -->
+    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\5_TestProjectsAnalyzers.editorconfig" Condition="'$(IsTestProject)' == 'true'" />
+  </ItemGroup>
+
+  <!-- Banned Symbols -->
+  <PropertyGroup>
+    <IncludeDefaultBannedSymbols Condition="$(IncludeDefaultBannedSymbols) == ''">true</IncludeDefaultBannedSymbols>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <AdditionalFiles Include="$(MSBuildThisFileDirectory)\..\files\BannedSymbols.txt"
+      Condition="$(IncludeDefaultBannedSymbols) == 'true'"
+      Visible="false" />
+  </ItemGroup>
+</Project>

src/buildMultiTargeting/Workleap.DotNet.CodingStandards.props

@@ -0,0 +1,3 @@
+<Project>
+    <Import Project="$(MSBuildThisFileDirectory)\..\build\Workleap.DotNet.CodingStandards.props" />
+</Project>

src/buildMultiTargeting/Workleap.DotNet.CodingStandards.targets

@@ -0,0 +1,3 @@
+<Project>
+    <Import Project="$(MSBuildThisFileDirectory)\..\build\Workleap.DotNet.CodingStandards.targets" />
+</Project>

src/buildTransitive/Workleap.DotNet.CodingStandards.props

@@ -1,29 +1,3 @@
 <Project>
-  <PropertyGroup>
-    <ReportAnalyzer Condition="'$(ReportAnalyzer)' == ''">true</ReportAnalyzer>
-    <Features Condition="'$(Features)' == ''">strict</Features>
-    <Deterministic Condition="'$(Deterministic)' == ''">true</Deterministic>
-    <EnableNETAnalyzers Condition="'$(EnableNETAnalyzers)' == ''">true</EnableNETAnalyzers>
-    <AnalysisLevel Condition="'$(AnalysisLevel)' == ''">latest-all</AnalysisLevel>
-    <EnforceCodeStyleInBuild Condition="'$(EnforceCodeStyleInBuild)' == ''">true</EnforceCodeStyleInBuild>
-
-    <!-- https://learn.microsoft.com/en-us/nuget/release-notes/nuget-5.5#summary-whats-new-in-55 -->
-    <RestoreUseStaticGraphEvaluation Condition="'$(RestoreUseStaticGraphEvaluation)' == ''">true</RestoreUseStaticGraphEvaluation>
-
-    <!-- Enable ContinuousIntegrationBuild when running on CI -->
-    <ContinuousIntegrationBuild Condition="'$(TF_BUILD)' == 'true'">true</ContinuousIntegrationBuild>
-    <ContinuousIntegrationBuild Condition="'$(GITHUB_ACTIONS)' == 'true'">true</ContinuousIntegrationBuild>
-    <ContinuousIntegrationBuild Condition="'$(GITLAB_CI)' == 'true'">true</ContinuousIntegrationBuild>
-    <ContinuousIntegrationBuild Condition="'$(CI)' == 'true'">true</ContinuousIntegrationBuild>
-    <ContinuousIntegrationBuild Condition="'$(TEAMCITY_VERSION)' != ''">true</ContinuousIntegrationBuild>
-
-    <!-- TreatWarningsAsErrors is enabled for release builds, unless explicitly set -->
-    <TreatWarningsAsErrors Condition="'$(Configuration)' == 'Release' AND '$(TreatWarningsAsErrors)' == ''">true</TreatWarningsAsErrors>
-
-    <!-- https://devblogs.microsoft.com/visualstudio/vs-toolbox-accelerate-your-builds-of-sdk-style-net-projects/ -->
-    <AccelerateBuildsInVisualStudio Condition="'$(AccelerateBuildsInVisualStudio)' == ''">true</AccelerateBuildsInVisualStudio>
-
-    <!-- GenerateDocumentationFile must be set to true for IDE0005 (Remove unnecessary usings/imports) to work -->
-    <GenerateDocumentationFile Condition="'$(GenerateDocumentationFile)' == ''">true</GenerateDocumentationFile>
-  </PropertyGroup>
-</Project>
+    <Import Project="$(MSBuildThisFileDirectory)\..\build\Workleap.DotNet.CodingStandards.props" />
+</Project>

src/buildTransitive/Workleap.DotNet.CodingStandards.targets

@@ -1,31 +1,3 @@
 <Project>
-  <!-- Register the EditorConfig files to the project -->
-  <!-- Remember that a particular .NET analysis rule can only be configured once across all imported global EditorConfig files -->
-  <ItemGroup>
-    <!-- Basic EditorConfig settings such as encoding, indentation, etc. -->
-    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\1_FileDefaults.editorconfig" />
-
-    <!-- C# code style -->
-    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\2_CodeStyle.editorconfig" />
-
-    <!-- .NET analyzers configuration for all projects, enforcing C# code style, quality, performance and security -->
-    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\3_AllProjectsAnalyzers.editorconfig" />
-
-    <!-- Configure ReSharper analyzers that overlaps with built-in .NET analyzers (only appears in Rider and VisualStudio IDEs) -->
-    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\4_ReSharperAnalyzers.editorconfig" />
-
-    <!-- .NET analyzers configuration only for test projects -->
-    <EditorConfigFiles Include="$(MSBuildThisFileDirectory)\..\files\5_TestProjectsAnalyzers.editorconfig" Condition="'$(IsTestProject)' == 'true'" />
-  </ItemGroup>
-
-  <!-- Banned Symbols -->
-  <PropertyGroup>
-    <IncludeDefaultBannedSymbols Condition="$(IncludeDefaultBannedSymbols) == ''">true</IncludeDefaultBannedSymbols>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <AdditionalFiles Include="$(MSBuildThisFileDirectory)\..\files\BannedSymbols.txt"
-      Condition="$(IncludeDefaultBannedSymbols) == 'true'"
-      Visible="false" />
-  </ItemGroup>
-</Project>
+    <Import Project="$(MSBuildThisFileDirectory)\..\build\Workleap.DotNet.CodingStandards.targets" />
+</Project>

tests/Workleap.DotNet.CodingStandards.Tests/Helpers/PathHelpers.cs

@@ -0,0 +1,17 @@
+using Meziantou.Framework;
+
+namespace Workleap.DotNet.CodingStandards.Tests.Helpers;
+
+internal static class PathHelpers
+{
+    public static FullPath GetRootDirectory()
+    {
+        var directory = FullPath.CurrentDirectory();
+        while (!Directory.Exists(directory / ".git"))
+        {
+            directory = directory.Parent;
+        }
+
+        return directory;
+    }
+}

tests/Workleap.DotNet.CodingStandards.Tests/Helpers/ProjectBuilder.cs

@@ -0,0 +1,103 @@
+using Meziantou.Framework;
+using System.Xml.Linq;
+using Xunit.Abstractions;
+using System.Text.Json;
+using CliWrap;
+
+namespace Workleap.DotNet.CodingStandards.Tests.Helpers;
+
+internal sealed class ProjectBuilder : IAsyncDisposable
+{
+    private const string SarifFileName = "BuildOutput.sarif";
+
+    private readonly TemporaryDirectory _directory;
+    private readonly ITestOutputHelper _testOutputHelper;
+
+    public ProjectBuilder(PackageFixture fixture, ITestOutputHelper testOutputHelper)
+    {
+        _testOutputHelper = testOutputHelper;
+
+        _directory = TemporaryDirectory.Create();
+        _directory.CreateTextFile("NuGet.config", $"""
+                <configuration>
+                  <config>
+                    <add key="globalPackagesFolder" value="{fixture.PackageDirectory}/packages" />
+                  </config>
+                  <packageSources>
+                    <clear />
+                    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
+                    <add key="TestSource" value="{fixture.PackageDirectory}" />
+                  </packageSources>
+                  <packageSourceMapping>
+                    <packageSource key="nuget.org">
+                        <package pattern="*" />
+                    </packageSource>
+                    <packageSource key="TestSource">
+                        <package pattern="Workleap.DotNet.CodingStandards" />
+                    </packageSource>
+                  </packageSourceMapping>
+                </configuration>
+                """);
+
+        File.Copy(PathHelpers.GetRootDirectory() / "global.json", _directory.FullPath / "global.json");
+    }
+
+    public ProjectBuilder AddFile(string relativePath, string content)
+    {
+        File.WriteAllText(_directory.FullPath / relativePath, content);
+        return this;
+    }
+
+    public ProjectBuilder AddCsprojFile(Dictionary<string, string> properties = null)
+    {
+        var element = new XElement("PropertyGroup");
+        if (properties != null)
+        {
+            foreach (var prop in properties)
+            {
+                element.Add(new XElement(prop.Key), prop.Value);
+            }
+        }
+
+        var content = $"""
+                <Project Sdk="Microsoft.NET.Sdk">
+                  <PropertyGroup>
+                    <OutputType>exe</OutputType>
+                    <TargetFramework>net$(NETCoreAppMaximumVersion)</TargetFramework>
+                    <ImplicitUsings>enable</ImplicitUsings>
+                    <Nullable>enable</Nullable>
+                    <ErrorLog>{SarifFileName},version=2.1</ErrorLog>
+                  </PropertyGroup>
+                  {element}
+
+                  <ItemGroup>
+                    <PackageReference Include="Workleap.DotNet.CodingStandards" Version="*" />
+                  </ItemGroup>
+                </Project>
+                """;
+
+        File.WriteAllText(_directory.FullPath / "test.csproj", content);
+        return this;
+    }
+
+    public async Task<SarifFile> BuildAndGetOutput(string[] buildArguments = null)
+    {
+        var result = await Cli.Wrap("dotnet")
+            .WithWorkingDirectory(_directory.FullPath)
+            .WithArguments(["build", .. (buildArguments ?? [])])
+            .WithEnvironmentVariables(env => env.Set("CI", null).Set("GITHUB_ACTIONS", null))
+            .WithStandardOutputPipe(PipeTarget.ToDelegate(_testOutputHelper.WriteLine))
+            .WithStandardErrorPipe(PipeTarget.ToDelegate(_testOutputHelper.WriteLine))
+            .WithValidation(CommandResultValidation.None)
+            .ExecuteAsync();
+
+        _testOutputHelper.WriteLine("Process exit code: " + result.ExitCode);
+
+        var bytes = File.ReadAllBytes(_directory.FullPath / SarifFileName);
+        var sarif = JsonSerializer.Deserialize<SarifFile>(bytes);
+        _testOutputHelper.WriteLine("Sarif result:\n" + string.Join("\n", sarif.AllResults().Select(r => r.ToString())));
+        return sarif;
+    }
+
+    public ValueTask DisposeAsync() => _directory.DisposeAsync();
+}

tests/Workleap.DotNet.CodingStandards.Tests/Helpers/SarifFile.cs

@@ -0,0 +1,16 @@
+using System.Text.Json.Serialization;
+
+namespace Workleap.DotNet.CodingStandards.Tests.Helpers;
+
+internal sealed class SarifFile
+{
+    [JsonPropertyName("runs")]
+    public SarifFileRun[] Runs { get; set; }
+
+    public IEnumerable<SarifFileRunResult> AllResults() => Runs.SelectMany(r => r.Results);
+
+    public bool HasError() => AllResults().Any(r => r.Level == "error");
+    public bool HasError(string ruleId) => AllResults().Any(r => r.Level == "error" && r.RuleId == ruleId);
+    public bool HasWarning(string ruleId) => AllResults().Any(r => r.Level == "warning" && r.RuleId == ruleId);
+    public bool HasNote(string ruleId) => AllResults().Any(r => r.Level == "note" && r.RuleId == ruleId);
+}

tests/Workleap.DotNet.CodingStandards.Tests/Helpers/SarifFileRun.cs

@@ -0,0 +1,9 @@
+using System.Text.Json.Serialization;
+
+namespace Workleap.DotNet.CodingStandards.Tests.Helpers;
+
+internal sealed class SarifFileRun
+{
+    [JsonPropertyName("results")]
+    public SarifFileRunResult[] Results { get; set; }
+}

tests/Workleap.DotNet.CodingStandards.Tests/Helpers/SarifFileRunResult.cs

@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+
+namespace Workleap.DotNet.CodingStandards.Tests.Helpers;
+
+internal sealed class SarifFileRunResult
+{
+    [JsonPropertyName("ruleId")]
+    public string RuleId { get; set; }
+
+    [JsonPropertyName("level")]
+    public string Level { get; set; }
+
+    [JsonPropertyName("message")]
+    public SarifFileRunResultMessage Message { get; set; }
+
+    public override string ToString()
+    {
+        return $"{Level}:{RuleId} {Message}";
+    }
+}

tests/Workleap.DotNet.CodingStandards.Tests/Helpers/SarifFileRunResultMessage.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace Workleap.DotNet.CodingStandards.Tests.Helpers;
+
+internal sealed class SarifFileRunResultMessage
+{
+    [JsonPropertyName("text")]
+    public string Text { get; set; }
+
+    public override string ToString()
+    {
+        return Text;
+    }
+}