diff --git a/ApiDocs.Console/CommandLineOptions.cs b/ApiDocs.Console/CommandLineOptions.cs index 624b52f..7ea562b 100644 --- a/ApiDocs.Console/CommandLineOptions.cs +++ b/ApiDocs.Console/CommandLineOptions.cs @@ -30,6 +30,7 @@ namespace ApiDocs.ConsoleApp using System.Linq; using ApiDocs.ConsoleApp.Auth; using ApiDocs.Validation; + using Validation.Config; using ApiDocs.Validation.Writers; using CommandLine; using CommandLine.Text; @@ -47,7 +48,8 @@ class CommandLineOptions public const string VerbAbout = "about"; public const string VerbCheckAll = "check-all"; public const string VerbPublishMetadata = "publish-edmx"; - + public const string VerbTestWithConfig = "use-config"; + [VerbOption(VerbPrint, HelpText="Print files, resources, and methods discovered in the documentation.")] public PrintOptions PrintVerbOptions { get; set; } @@ -60,6 +62,9 @@ class CommandLineOptions [VerbOption(VerbCheckAll, HelpText = "Check for errors in the documentation (links + resources + examples)")] public BasicCheckOptions CheckAllVerbs { get; set; } + [VerbOption(VerbTestWithConfig, HelpText="Check for errors in the documentation using a configuration files")] + public CheckWithConfigOptions CheckConfigFileVerb { get; set; } + [VerbOption(VerbService, HelpText = "Check for errors between the documentation and service.")] public CheckServiceOptions CheckServiceVerb { get; set; } @@ -85,7 +90,22 @@ public string GetUsage(string verb) class BaseOptions { - [Option("log", HelpText="Write the console output to file.")] +#if DEBUG + [Option("debug", HelpText="Launch the debugger before doing anything interesting")] + public bool AttachDebugger { get; set; } +#endif + + public virtual bool HasRequiredProperties(out string[] missingArguments) + { + missingArguments = new string[0]; + return true; + } + + } + + class CommandLineBaseOptions : BaseOptions + { + [Option("log", HelpText = "Write the console output to file.")] public string LogFile { get; set; } [Option("ignore-warnings", HelpText = "Ignore warnings as errors for pass rate.")] @@ -94,10 +114,10 @@ class BaseOptions [Option("silence-warnings", HelpText = "Don't print warnings to the screen or consider them errors")] public bool SilenceWarnings { get; set; } - [Option("appveyor-url", HelpText="Specify the AppVeyor Build Worker API URL for output integration")] + [Option("appveyor-url", HelpText = "Specify the AppVeyor Build Worker API URL for output integration")] public string AppVeyorServiceUrl { get; set; } - [Option("ignore-errors", HelpText="Prevent errors from generating a non-zero return code.")] + [Option("ignore-errors", HelpText = "Prevent errors from generating a non-zero return code.")] public bool IgnoreErrors { get; set; } [Option("parameters", HelpText = "Specify additional page variables that are used by the publishing engine. URL encoded: key=value&key2=value2.")] @@ -106,9 +126,8 @@ class BaseOptions [Option("print-failures-only", HelpText = "Only prints test failures to the console.")] public bool PrintFailuresOnly { get; set; } - - - public Dictionary PageParameterDict { + public Dictionary PageParameterDict + { get { if (string.IsNullOrEmpty(AdditionalPageParameters)) @@ -125,22 +144,50 @@ public Dictionary PageParameterDict { } } -#if DEBUG - [Option("debug", HelpText="Launch the debugger before doing anything interesting")] - public bool AttachDebugger { get; set; } -#endif - public virtual bool HasRequiredProperties(out string[] missingArguments) + /// + /// Convert command line parameters into the parameters for our validator + /// + /// + internal virtual Validation.Config.ApiDocsParameters GetParameters() { - missingArguments = new string[0]; - return true; + var config = new Validation.Config.ApiDocsParameters(); + config.Reporting = new ReportingParameters(); + config.Reporting.Console = new Validation.Config.ReportingEngineParameters() { Path = LogFile }; + config.Severity = new Validation.Config.SeverityParameters(); + if (IgnoreWarnings) + { + config.Severity.Warnings = Validation.Config.SeverityLevel.Ignored; + } + if (IgnoreErrors) + { + config.Severity.Errors = Validation.Config.SeverityLevel.Ignored; + } + + if (!string.IsNullOrEmpty(AppVeyorServiceUrl)) + { + config.Reporting.Appveyor = new Validation.Config.ReportingEngineParameters() + { + Url = AppVeyorServiceUrl + }; + } + if (!string.IsNullOrEmpty(AdditionalPageParameters)) + { + config.PageParameters = PageParameterDict; + } + if (PrintFailuresOnly) + { + config.Reporting.Console.Level = Validation.Config.LogLevel.ErrorsOnly; + } + + return config; } } /// /// Command line options for any verbs that work with a documentation set. /// - class DocSetOptions : BaseOptions + class DocSetOptions : CommandLineBaseOptions { internal const string PathArgument = "path"; internal const string VerboseArgument = "verbose"; @@ -167,6 +214,37 @@ public override bool HasRequiredProperties(out string[] missingArguments) missingArguments = new string[0]; return true; } + + internal override ApiDocsParameters GetParameters() + { + var config = base.GetParameters(); + if (EnableVerboseOutput) + { + config.Reporting.Console.Level = LogLevel.Verbose; + } + return config; + } + } + + class CheckWithConfigOptions : BaseOptions + { + [Option("file", HelpText = "Path to the configuration file that should be used.")] + public string ConfigurationFile { get; set; } + + [Option("set", HelpText="Run only the specified set from the configuration files")] + public string ChosenSet { get; set; } + + public override bool HasRequiredProperties(out string[] missingArguments) + { + if (string.IsNullOrEmpty(ConfigurationFile)) + { + missingArguments = new string[] { "config" }; + return false; + }; + + missingArguments = null; + return true; + } } class CheckMetadataOptions : DocSetOptions @@ -174,6 +252,18 @@ class CheckMetadataOptions : DocSetOptions [Option("metadata", HelpText = "Path or URL for the service metadata CSDL")] public string ServiceMetadataLocation { get; set; } + + internal override ApiDocsParameters GetParameters() + { + var config = base.GetParameters(); + + config.CheckMetadataParameters = new CheckMetadataActionParameters + { + SchemaUrls = new string[] { ServiceMetadataLocation } + }; + + return config; + } } class PrintOptions : DocSetOptions @@ -229,6 +319,44 @@ class BasicCheckOptions : DocSetOptions [Option("link-case-match", HelpText = "Require the CaSe of relative links within the content to match the filenames.")] public bool RequireFilenameCaseMatch { get; set; } + + internal override ApiDocsParameters GetParameters() + { + var config = base.GetParameters(); + config.SharedActionParameters = new SharedActionParameters(); + if (!string.IsNullOrEmpty(MethodName)) + { + config.SharedActionParameters.MethodFilter = MethodName; + } + if (!string.IsNullOrEmpty(FileName)) + { + config.SharedActionParameters.FilenameFilter = FileName; + } + if (ForceAllScenarios) + { + config.SharedActionParameters.RunAllScenarios = true; + } + if (RelaxStringTypeValidation) + { + config.SharedActionParameters.RelaxStringValidation = true; + } + if (!string.IsNullOrEmpty(FilesChangedFromOriginalBranch)) + { + config.PullRequests = config.PullRequests ?? new PullRequestParameters(); + config.PullRequests.TargetBranch = FilesChangedFromOriginalBranch; + } + if (!string.IsNullOrEmpty(GitExecutablePath)) + { + config.GitExecutablePath = GitExecutablePath; + } + if (RequireFilenameCaseMatch) + { + config.CheckLinksParameters = config.CheckLinksParameters ?? new CheckLinksActionParameters(); + config.CheckLinksParameters.LinksAreCaseSensitive = RequireFilenameCaseMatch; + } + + return config; + } } class CheckServiceOptions : BasicCheckOptions @@ -377,6 +505,38 @@ public override bool HasRequiredProperties(out string[] missingArguments) return missingArguments.Length == 0; } + internal override ApiDocsParameters GetParameters() + { + var config = base.GetParameters(); + + config.CheckServiceParameters = new CheckServiceActionParameters(); + if (!string.IsNullOrEmpty(AccountName)) + { + config.CheckServiceParameters.Accounts = new string[] { AccountName }; + } + if (PauseBetweenRequests) + { + config.CheckServiceParameters.PauseBetweenRequests = PauseBetweenRequests; + } + if (!string.IsNullOrEmpty(AdditionalHeaders)) + { + config.CheckServiceParameters.AdditionalHeaders = AdditionalHeaders.Split(new char[] { '|' }, StringSplitOptions.RemoveEmptyEntries); + } + if (!string.IsNullOrEmpty(ODataMetadataLevel)) + { + config.CheckServiceParameters.MetadataLevel = ODataMetadataLevel; + } + if (!string.IsNullOrEmpty(BranchName)) + { + config.CheckServiceParameters.BranchName = BranchName; + } + if (ParallelTests) + { + config.CheckServiceParameters.RunTestsInParallel = ParallelTests; + } + return config; + } + } class PublishMetadataOptions : DocSetOptions diff --git a/ApiDocs.Console/Program.cs b/ApiDocs.Console/Program.cs index e725cc4..5b7a5b3 100644 --- a/ApiDocs.Console/Program.cs +++ b/ApiDocs.Console/Program.cs @@ -42,7 +42,7 @@ namespace ApiDocs.ConsoleApp using Validation.Writers; using CommandLine; using Newtonsoft.Json; - + using Validation.Config; class Program { @@ -53,9 +53,6 @@ class Program public static readonly BuildWorkerApi BuildWorker = new BuildWorkerApi(); public static AppConfigFile CurrentConfiguration { get; private set; } - // Set to true to disable returning an error code when the app exits. - private static bool IgnoreErrors { get; set; } - private static List DiscoveredUndocumentedProperties = new List(); static void Main(string[] args) @@ -84,7 +81,6 @@ static void Main(string[] args) Exit(failure: true); } - IgnoreErrors = verbOptions.IgnoreErrors; #if DEBUG if (verbOptions.AttachDebugger) { @@ -106,8 +102,14 @@ static void Main(string[] args) } } - private static void SetStateFromOptions(BaseOptions verbOptions) + private static void SetStateFromOptions(BaseOptions options) { + CommandLineBaseOptions verbOptions = options as CommandLineBaseOptions; + if (null == verbOptions) + { + return; + } + if (!string.IsNullOrEmpty(verbOptions.AppVeyorServiceUrl)) { BuildWorker.UrlEndPoint = new Uri(verbOptions.AppVeyorServiceUrl); @@ -166,13 +168,13 @@ private static async Task RunInvokedMethodAsync(CommandLineOptions origCommandLi { var error = new ValidationError(ValidationErrorCode.MissingRequiredArguments, null, "Command line is missing required arguments: {0}", missingProps.ComponentsJoinedByString(", ")); FancyConsole.WriteLine(origCommandLineOpts.GetUsage(invokedVerb)); - await WriteOutErrorsAndFinishTestAsync(new ValidationError[] { error }, options.SilenceWarnings, printFailuresOnly: options.PrintFailuresOnly); + await WriteOutErrorsAndFinishTestAsync(new ValidationError[] { error }, false); Exit(failure: true); } LoadCurrentConfiguration(options as DocSetOptions); - bool returnSuccess = true; + Validation.Logger.TestResult result = Validation.Logger.TestResult.NotStarted; switch (invokedVerb) { @@ -180,23 +182,26 @@ private static async Task RunInvokedMethodAsync(CommandLineOptions origCommandLi await PrintDocInformationAsync((PrintOptions)options); break; case CommandLineOptions.VerbCheckLinks: - returnSuccess = await CheckLinksAsync((BasicCheckOptions)options); + result = await CheckLinksAsync((BasicCheckOptions)options); break; case CommandLineOptions.VerbDocs: - returnSuccess = await CheckDocsAsync((BasicCheckOptions)options); + result = await CheckDocsAsync((BasicCheckOptions)options); break; case CommandLineOptions.VerbCheckAll: - returnSuccess = await CheckDocsAllAsync((BasicCheckOptions)options); - break; - case CommandLineOptions.VerbService: - returnSuccess = await CheckServiceAsync((CheckServiceOptions)options); - break; - case CommandLineOptions.VerbPublish: - returnSuccess = await PublishDocumentationAsync((PublishOptions)options); + result = await CheckDocsAllAsync((BasicCheckOptions)options); break; - case CommandLineOptions.VerbPublishMetadata: - returnSuccess = await PublishMetadataAsync((PublishMetadataOptions)options); + case CommandLineOptions.VerbTestWithConfig: + result = await CheckWithConfigOptionsAsync((CheckWithConfigOptions)options); break; + //case CommandLineOptions.VerbService: + // result = await CheckServiceAsync((CheckServiceOptions)options); + // break; + //case CommandLineOptions.VerbPublish: + // result = await PublishDocumentationAsync((PublishOptions)options); + // break; + //case CommandLineOptions.VerbPublishMetadata: + // result = await PublishMetadataAsync((PublishMetadataOptions)options); + // break; case CommandLineOptions.VerbMetadata: await CheckServiceMetadataAsync((CheckMetadataOptions)options); break; @@ -206,7 +211,64 @@ private static async Task RunInvokedMethodAsync(CommandLineOptions origCommandLi break; } - Exit(failure: !returnSuccess); + Exit(result); + } + + /// + /// Load the configuration file and perform the requested validations + /// + /// + /// + private static async Task CheckWithConfigOptionsAsync(CheckWithConfigOptions options) + { + var config = DocSet.TryLoadConfigurationFiles(options.ConfigurationFile).FirstOrDefault(); + if (null == config) + { + return Validation.Logger.TestResult.NothingToTest; + } + + Console.WriteLine($"Found {config.Sets.Count} sets in the configuration file."); + + Validation.Logger.TestResult overallResults = Validation.Logger.TestResult.Running; + + foreach(var set in config.Sets) + { + // Skip the set if a chosen set was specified and this isn't it + if (options.ChosenSet != null && !options.ChosenSet.Equals(set.Key, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + Validation.Config.DocSetConfiguration setParams = set.Value; + setParams.Name = set.Key; + + // Load the docs for this set + DocSet docs = new DocSet(setParams.RelativePath); + ValidationError[] detectedErrors = null; + await Task.Run(() => { docs.ScanDocumentation("", out detectedErrors); }); + + Validation.Logger.TestResult results = Validation.Logger.TestResult.Running; + + // Execute the specified actions + if (setParams.Actions.CheckLinks != null) + { + var linkResults = await CheckLinksAsync(docs, config.DefaultParameters); + results = (new Validation.Logger.TestResult[] { linkResults, results }).Min(); + } + + if (setParams.Actions.CheckDocs != null) + { + var docResults = await CheckDocsAsync(docs, config.DefaultParameters); + results = (new Validation.Logger.TestResult[] { docResults, results }).Min(); + } + + Console.WriteLine($"Set {setParams.Name} has completed. Overall results: {results}."); + + overallResults = (new Validation.Logger.TestResult[] { overallResults, results }).Min(); + } + + Console.WriteLine($"Tests are finished. Overall result: {overallResults}."); + return overallResults; } /// @@ -216,17 +278,19 @@ private static async Task RunInvokedMethodAsync(CommandLineOptions origCommandLi /// /// /// - private static async Task CheckDocsAllAsync(BasicCheckOptions options) + private static async Task CheckDocsAllAsync(BasicCheckOptions options) { var docset = await GetDocSetAsync(options); if (null == docset) - return false; + { + return Validation.Logger.TestResult.NothingToTest; + } var checkLinksResult = await CheckLinksAsync(options, docset); var checkDocsResults = await CheckDocsAsync(options, docset); - return checkLinksResult && checkDocsResults; + return (Validation.Logger.TestResult)Math.Min((int)checkLinksResult, (int)checkDocsResults); } private static void PrintAboutMessage() @@ -464,59 +528,30 @@ private static async Task PrintAccountsAsync(PrintOptions options, DocSet docset /// /// /// - private static async Task CheckLinksAsync(BasicCheckOptions options, DocSet docs = null) + private static async Task CheckLinksAsync(BasicCheckOptions options, DocSet docs = null) { - const string testName = "Check-links"; var docset = docs ?? await GetDocSetAsync(options); - if (null == docset) - return false; - - - string[] interestingFiles = null; - if (!string.IsNullOrEmpty(options.FilesChangedFromOriginalBranch)) - { - GitHelper helper = new GitHelper(options.GitExecutablePath, options.DocumentationSetPath); - interestingFiles = helper.FilesChangedFromBranch(options.FilesChangedFromOriginalBranch); - } - - TestReport.StartTest(testName); - - ValidationError[] errors; - docset.ValidateLinks(options.EnableVerboseOutput, interestingFiles, out errors, options.RequireFilenameCaseMatch); - - foreach (var error in errors) { - MessageCategory category; - if (error.IsWarning) - category = MessageCategory.Warning; - else if (error.IsError) - category = MessageCategory.Error; - else - category = MessageCategory.Information; - - string message = string.Format("{1}: {0}", error.Message.FirstLineOnly(), error.Code); - await TestReport.LogMessageAsync(message, category); + return Validation.Logger.TestResult.NothingToTest; } - return await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, successMessage: "No link errors detected.", testName: testName, printFailuresOnly: options.PrintFailuresOnly); + var parameters = options.GetParameters(); + return await CheckLinksAsync(docset, parameters); } - /// - /// Find the first instance of a method with a particular name in the docset. - /// - /// - /// - /// - private static MethodDefinition LookUpMethod(DocSet docset, string methodName) + private static async Task CheckLinksAsync(DocSet docset, Validation.Config.ApiDocsParameters parameters) { - var query = from m in docset.Methods - where m.Identifier.Equals(methodName, StringComparison.OrdinalIgnoreCase) - select m; + if (null == docset) throw new ArgumentNullException(nameof(docset)); + + var tester = new Validation.Logger.TestEngine(parameters, docset.SourceFolderPath); + var result = await ValidationProcessor.CheckLinksAsync(parameters, docset, tester); + await tester.CompleteAsync(); - return query.FirstOrDefault(); + return result; } + /// /// Returns a collection of methods matching the string query. This can either be the /// literal name of the method of a wildcard match for the method name. @@ -540,160 +575,29 @@ where m.Identifier.IsWildcardMatch(wildcardPattern) /// /// /// - private static async Task CheckDocsAsync(BasicCheckOptions options, DocSet docs = null) + private static async Task CheckDocsAsync(BasicCheckOptions options, DocSet docs = null) { var docset = docs ?? await GetDocSetAsync(options); - if (null == docset) - return false; - - FancyConsole.WriteLine(); - - var resultStructure = await CheckDocStructure(options, docset); - - var resultMethods = await CheckMethodsAsync(options, docset); - CheckResults resultExamples = new CheckResults(); - if (string.IsNullOrEmpty(options.MethodName)) - { - resultExamples = await CheckExamplesAsync(options, docset); - } - - var combinedResults = resultMethods + resultExamples + resultStructure; - - if (options.IgnoreWarnings) - { - combinedResults.ConvertWarningsToSuccess(); - } - - combinedResults.PrintToConsole(); - - return combinedResults.FailureCount == 0; - } - - private static async Task CheckDocStructure(BasicCheckOptions options, DocSet docset) - { - var results = new CheckResults(); - TestReport.StartTest("Verify document structure"); - List detectedErrors = new List(); - foreach (var doc in docset.Files) + if (null == docset) { - detectedErrors.AddRange(doc.CheckDocumentStructure()); + return Validation.Logger.TestResult.NothingToTest; } - await WriteOutErrorsAndFinishTestAsync(detectedErrors, options.SilenceWarnings, " ", "Passed.", false, "Verify document structure", "Warnings detected", "Errors detected", printFailuresOnly: options.PrintFailuresOnly); - results.IncrementResultCount(detectedErrors); - return results; + var parameters = options.GetParameters(); + return await CheckDocsAsync(docset, parameters); } - /// - /// Perform an internal consistency check on the examples defined in the documentation. Prints - /// the results of the tests to the console. - /// - /// - /// - /// - private static async Task CheckExamplesAsync(BasicCheckOptions options, DocSet docset) + private async static Task CheckDocsAsync(DocSet docs, ApiDocsParameters parameters) { - var results = new CheckResults(); - - foreach (var doc in docset.Files) - { - if (doc.Examples.Length == 0) - { - continue; - } - - FancyConsole.WriteLine(FancyConsole.ConsoleHeaderColor, "Checking examples in \"{0}\"...", doc.DisplayName); - - foreach (var example in doc.Examples) - { - if (example.Metadata == null) - continue; - if (example.Language != CodeLanguage.Json) - continue; - - var testName = string.Format("check-example: {0}", example.Metadata.MethodName, example.Metadata.ResourceType); - TestReport.StartTest(testName, doc.DisplayName); - - ValidationError[] errors; - docset.ResourceCollection.ValidateJsonExample(example.Metadata, example.SourceExample, out errors, new ValidationOptions { RelaxedStringValidation = options.RelaxStringTypeValidation }); - - await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, " ", "Passed.", false, testName, "Warnings detected", "Errors detected", printFailuresOnly: options.PrintFailuresOnly); - results.IncrementResultCount(errors); - } - } - - return results; - } - - /// - /// Performs an internal consistency check on the methods (requests/responses) in the documentation. - /// Prints the results of the tests to the console. - /// - /// - /// - /// - private static async Task CheckMethodsAsync(BasicCheckOptions options, DocSet docset) - { - MethodDefinition[] methods = FindTestMethods(options, docset); - CheckResults results = new CheckResults(); - - foreach (var method in methods) - { - var testName = "API Request: " + method.Identifier; - - TestReport.StartTest(testName, method.SourceFile.DisplayName, skipPrintingHeader: options.PrintFailuresOnly); - - if (string.IsNullOrEmpty(method.ExpectedResponse)) - { - await TestReport.FinishTestAsync(testName, TestOutcome.Failed, "Null response where one was expected.", printFailuresOnly: options.PrintFailuresOnly); - results.FailureCount++; - continue; - } - - var parser = new HttpParser(); - - - ValidationError[] errors; - try - { - var expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse); - - method.ValidateResponse(expectedResponse, null, null, out errors, new ValidationOptions { RelaxedStringValidation = options.RelaxStringTypeValidation }); - } - catch (Exception ex) - { - errors = new ValidationError[] { new ValidationError(ValidationErrorCode.ExceptionWhileValidatingMethod, method.SourceFile.DisplayName, ex.Message) }; - } + if (null == docs) throw new ArgumentNullException(nameof(docs)); - await WriteOutErrorsAndFinishTestAsync(errors, options.SilenceWarnings, " ", "Passed.", false, testName, "Warnings detected", "Errors detected", printFailuresOnly: options.PrintFailuresOnly); - results.IncrementResultCount(errors); - } - - return results; + var tester = new Validation.Logger.TestEngine(parameters, docs.SourceFolderPath); + var result = await ValidationProcessor.CheckDocsAsync(parameters, docs, tester); + await tester.CompleteAsync(); + return result; } - private static DocFile[] GetSelectedFiles(BasicCheckOptions options, DocSet docset) - { - List files = new List(); - if (!string.IsNullOrEmpty(options.FilesChangedFromOriginalBranch)) - { - GitHelper helper = new GitHelper(options.GitExecutablePath, options.DocumentationSetPath); - var changedFiles = helper.FilesChangedFromBranch(options.FilesChangedFromOriginalBranch); - - foreach (var filePath in changedFiles) - { - var file = docset.LookupFileForPath(filePath); - if (null != file) - files.Add(file); - } - } - else - { - files.AddRange(docset.Files); - } - return files.ToArray(); - } /// /// Parse the command line parameters into a set of methods that match the command line parameters. @@ -916,8 +820,6 @@ internal static void WriteValidationError(string indent, ValidationError error) /// that the actual requests come from the service instead of the documentation. Prints the errors to /// the console. /// - /// - /// private static async Task CheckServiceAsync(CheckServiceOptions options) { // See if we're supposed to run check-service on this branch (assuming we know what branch we're running on) @@ -1202,6 +1104,24 @@ private static void ConfigureAdditionalHeadersForAccount(CheckServiceOptions opt } } + public static void Exit(Validation.Logger.TestResult result) + { + switch(result) + { + case Validation.Logger.TestResult.Passed: + case Validation.Logger.TestResult.PassedWithWarnings: + Exit(failure: false); + break; + case Validation.Logger.TestResult.Failed: + Exit(failure: true); + break; + case Validation.Logger.TestResult.NothingToTest: + case Validation.Logger.TestResult.NotStarted: + Exit(failure: true, customExitCode: 99); + break; + } + } + private static void Exit(bool failure, int? customExitCode = null) { int exitCode = failure ? ExitCodeFailure : ExitCodeSuccess; @@ -1210,12 +1130,6 @@ private static void Exit(bool failure, int? customExitCode = null) exitCode = customExitCode.Value; } - if (IgnoreErrors) - { - FancyConsole.WriteLine("Ignoring errors and returning a successful exit code."); - exitCode = ExitCodeSuccess; - } - #if DEBUG Console.WriteLine("Exit code: " + exitCode); if (Debugger.IsAttached) diff --git a/ApiDocs.Validation/ApiDocs.Validation.csproj b/ApiDocs.Validation/ApiDocs.Validation.csproj index 9ea35ca..e85fb10 100644 --- a/ApiDocs.Validation/ApiDocs.Validation.csproj +++ b/ApiDocs.Validation/ApiDocs.Validation.csproj @@ -51,13 +51,17 @@ + - + + + + @@ -66,8 +70,14 @@ + + + + + + @@ -104,6 +114,8 @@ + + diff --git a/ApiDocs.Validation/Config/SetsConfigFile.cs b/ApiDocs.Validation/Config/SetsConfigFile.cs new file mode 100644 index 0000000..8f39eec --- /dev/null +++ b/ApiDocs.Validation/Config/SetsConfigFile.cs @@ -0,0 +1,255 @@ +/* + * Markdown Scanner + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the ""Software""), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +namespace ApiDocs.Validation.Config +{ + using Newtonsoft.Json; + using System.Collections.Generic; + using System.Runtime.Serialization; + + public class SetsConfigFile : ConfigFile + { + + [JsonProperty("sets")] + public Dictionary Sets { get; set; } + + [JsonProperty("default-parameters")] + public ApiDocsParameters DefaultParameters { get; set; } + + public override bool IsValid + { + get + { + return Sets != null; + } + } + } + + public class DocSetConfiguration + { + [JsonIgnore] + public string Name { get; set; } + + [JsonProperty("path")] + public string RelativePath { get; set; } + + [JsonProperty("actions")] + public DocSetActions Actions { get; set; } + } + + public class ApiDocsParameters + { + [JsonProperty("reporting")] + public ReportingParameters Reporting { get; set; } + + [JsonProperty("severity")] + public SeverityParameters Severity { get; set; } + + [JsonProperty("shared")] + public SharedActionParameters SharedActionParameters { get; set; } + + [JsonProperty("check-docs")] + public CheckDocsActionParameters CheckDocsParameters { get; set; } + + [JsonProperty("check-links")] + public CheckLinksActionParameters CheckLinksParameters { get; set; } + + [JsonProperty("check-metadata")] + public CheckMetadataActionParameters CheckMetadataParameters { get; set; } + + [JsonProperty("check-service")] + public CheckServiceActionParameters CheckServiceParameters { get; set; } + + [JsonProperty("git-path")] + public string GitExecutablePath { get; set; } + + [JsonProperty("pull-requests")] + public PullRequestParameters PullRequests { get; set; } + + [JsonProperty("page-parameters")] + public Dictionary PageParameters { get; set; } + + } + + public class ReportingParameters + { + [JsonProperty("console")] + public ReportingEngineParameters Console { get; set; } + + [JsonProperty("appveyor")] + public ReportingEngineParameters Appveyor { get; set; } + + [JsonProperty("text-file")] + public ReportingEngineParameters TextFile { get; set; } + + [JsonProperty("http-tracer")] + public ReportingEngineParameters HttpTracer { get; set; } + } + + public class SharedActionParameters + { + [JsonProperty("method-filter")] + public string MethodFilter { get; set; } + + [JsonProperty("filename-filter")] + public string FilenameFilter { get; set; } + + [JsonProperty("run-all-scenarions")] + public bool RunAllScenarios { get; set; } + + [JsonProperty("relax-string-validation")] + public bool RelaxStringValidation { get; set; } + + } + + public class ReportingEngineParameters + { + [JsonProperty("path")] + public string Path { get; set; } + [JsonProperty("log-level")] + public LogLevel Level { get; set; } + [JsonProperty("url")] + public string Url { get; set; } + } + + public class SeverityParameters + { + [JsonProperty("errors")] + public SeverityLevel Errors { get; set; } + [JsonProperty("warnings")] + public SeverityLevel Warnings { get; set; } + [JsonProperty("messages")] + public SeverityLevel Messages { get; set; } + } + + + public class DocSetActions + { + [JsonProperty("check-links")] + public CheckLinksActionParameters CheckLinks { get; set; } + + [JsonProperty("check-docs")] + public CheckDocsActionParameters CheckDocs { get; set; } + + [JsonProperty("check-service")] + public CheckServiceActionParameters CheckService { get; set; } + + [JsonProperty("check-metadata")] + public CheckMetadataActionParameters CheckServiceMetadata { get; set; } + + //[JsonProperty("publish-docs")] + //public PublishDocsActionParameters PublishDocs { get; set; } + } + + public class CheckLinksActionParameters + { + [JsonProperty("links-case-sensitive")] + public bool LinksAreCaseSensitive { get; set; } + + } + + public class CheckDocsActionParameters : SharedActionParameters + { + [JsonProperty("check-structure")] + public bool ValidateStructure { get; set; } + + [JsonProperty("check-methods")] + public bool ValidateMethods { get; set; } + + [JsonProperty("check-examples")] + public bool ValidateExamples { get; set; } + + public CheckDocsActionParameters() + { + this.ValidateExamples = true; + this.ValidateMethods = true; + this.ValidateStructure = true; + } + + } + + public class CheckServiceActionParameters : SharedActionParameters + { + [JsonProperty("accounts")] + public string[] Accounts { get; set; } + + [JsonProperty("pause-between-requests")] + public bool PauseBetweenRequests { get; set; } + + [JsonProperty("headers")] + public string[] AdditionalHeaders { get; set; } + + [JsonProperty("metadata-level")] + public string MetadataLevel { get; set; } + + [JsonProperty("branch-name")] + public string BranchName { get; set; } + + [JsonProperty("run-in-parallel")] + public bool RunTestsInParallel { get; set; } + } + + public class CheckMetadataActionParameters + { + [JsonProperty("schema-urls")] + public string[] SchemaUrls { get; set; } + } + + public class PublishDocsActionParameters + { + public string OutputRelativePath { get; set; } + + } + + public class PullRequestParameters + { + [JsonProperty("pull-request-detector")] + public string PullRequestDetector { get; set; } + [JsonProperty("target-branch")] + public string TargetBranch { get; set; } + } + + public enum LogLevel + { + [EnumMember(Value = "default")] + Default = 0, + [EnumMember(Value = "errorsOnly")] + ErrorsOnly, + [EnumMember(Value = "verbose")] + Verbose + } + + public enum SeverityLevel + { + [EnumMember(Value = "default")] + Default = 0, + [EnumMember(Value = "critical")] + Critical, + [EnumMember(Value = "warn")] + Warning, + [EnumMember(Value = "ignore")] + Ignored + } +} diff --git a/ApiDocs.Validation/DocFile.cs b/ApiDocs.Validation/DocFile.cs index 8d5a7a8..8e39126 100644 --- a/ApiDocs.Validation/DocFile.cs +++ b/ApiDocs.Validation/DocFile.cs @@ -792,8 +792,8 @@ private void MergeParametersIntoCollection( } else if (addMissingParameters) { - Console.WriteLine($"Found property '{param.Name}' in markdown table that wasn't defined in '{resourceName}': {this.DisplayName}"); - detectedErrors.Add(new ValidationWarning(ValidationErrorCode.AdditionalPropertyDetected, this.DisplayName, $"Property '{param.Name}' found in markdown table but not in resource definition for '{resourceName}'.")); + //Console.WriteLine($"Found property '{param.Name}' in markdown table that wasn't defined in '{resourceName}': {this.DisplayName}"); + detectedErrors.Add(new UndocumentedPropertyWarning(this.DisplayName, param.Name, param.Type, resourceName, "Found in description table but not in the resource")); // The parameter didn't exist in the collection, so let's add it. collection.Add(param); @@ -801,8 +801,9 @@ private void MergeParametersIntoCollection( else { // Oops, we didn't find the property in the resource definition - Console.WriteLine($"Found property '{param.Name}' in markdown table that wasn't defined in '{resourceName}': {this.DisplayName}"); - detectedErrors.Add(new ValidationWarning(ValidationErrorCode.AdditionalPropertyDetected, this.DisplayName, $"Property '{param.Name}' found in markdown table but not in resource definition for '{resourceName}'.")); + //Console.WriteLine($"Found property '{param.Name}' in markdown table that wasn't defined in '{resourceName}': {this.DisplayName}"); + detectedErrors.Add(new UndocumentedPropertyWarning(this.DisplayName, param.Name, param.Type, resourceName, "Found in description table but not in the resource")); + //detectedErrors.Add(new ValidationWarning(ValidationErrorCode.AdditionalPropertyDetected, this.DisplayName, $"Property '{param.Name}' found in markdown table but not in resource definition for '{resourceName}'.")); } } } @@ -1079,7 +1080,7 @@ public bool ValidateNoBrokenLinks(bool includeWarnings, out ValidationError[] er { case LinkValidationResult.ExternalSkipped: if (includeWarnings) - foundErrors.Add(new ValidationWarning(ValidationErrorCode.LinkValidationSkipped, this.DisplayName, "Skipped validation of external link '[{1}]({0})'", link.Definition.url, link.Text)); + foundErrors.Add(new ValidationMessage(this.DisplayName, "Skipped validation of external link '[{1}]({0})'", link.Definition.url, link.Text)); break; case LinkValidationResult.FileNotFound: foundErrors.Add(new ValidationError(ValidationErrorCode.LinkDestinationNotFound, this.DisplayName, "FileNotFound: '[{1}]({0})'. {2}", link.Definition.url, link.Text, suggestion)); diff --git a/ApiDocs.Validation/DocSet.cs b/ApiDocs.Validation/DocSet.cs index d64ef32..356ae19 100644 --- a/ApiDocs.Validation/DocSet.cs +++ b/ApiDocs.Validation/DocSet.cs @@ -177,40 +177,62 @@ public static T[] TryLoadConfigurationFiles(string path) where T : ConfigFile { List validConfigurationFiles = new List(); - DirectoryInfo docSetDir = new DirectoryInfo(path); - if (!docSetDir.Exists) - return new T[0]; - - var jsonFiles = docSetDir.GetFiles("*.json", SearchOption.AllDirectories); - foreach (var file in jsonFiles) + if (File.Exists(path)) { - try + var configFile = LoadConfigFile(new FileInfo(path)); + if (null != configFile) { - using (var reader = file.OpenText()) - { - var config = JsonConvert.DeserializeObject(reader.ReadToEnd()); - if (null != config && config.IsValid) - { - config.LoadComplete(); - validConfigurationFiles.Add(config); - config.SourcePath = file.FullName; - } - } + validConfigurationFiles.Add(configFile); } - catch (JsonException ex) + else { - Logging.LogMessage(new ValidationWarning(ValidationErrorCode.JsonParserException, file.FullName, "JSON parser error: {0}", ex.Message)); + Logging.LogMessage(new ValidationWarning(ValidationErrorCode.JsonParserException, path, "Unable to read the configuration file specified.")); } - catch (Exception ex) + } + else if (Directory.Exists(path)) + { + DirectoryInfo docSetDir = new DirectoryInfo(path); + var jsonFiles = docSetDir.GetFiles("*.json", SearchOption.AllDirectories); + foreach (var file in jsonFiles) { - Logging.LogMessage(new ValidationWarning(ValidationErrorCode.JsonParserException, file.FullName, "Exception reading file: {0}", ex.Message)); + var configFile = LoadConfigFile(file); + if (null != configFile) + { + validConfigurationFiles.Add(configFile); + } } - } return validConfigurationFiles.ToArray(); } + private static T LoadConfigFile(FileInfo file) where T : ConfigFile + { + try + { + using (var reader = file.OpenText()) + { + var config = JsonConvert.DeserializeObject(reader.ReadToEnd()); + if (null != config && config.IsValid) + { + config.LoadComplete(); + config.SourcePath = file.FullName; + return config; + } + } + } + catch (JsonException ex) + { + Logging.LogMessage(new ValidationWarning(ValidationErrorCode.JsonParserException, file.FullName, "JSON parser error: {0}", ex.Message)); + } + catch (Exception ex) + { + Logging.LogMessage(new ValidationWarning(ValidationErrorCode.JsonParserException, file.FullName, "Exception reading file: {0}", ex.Message)); + } + return null; + } + + public static string ResolvePathWithUserRoot(string path) { if (path.StartsWith(string.Concat("~", Path.DirectorySeparatorChar.ToString()))) diff --git a/ApiDocs.Validation/Error/ValidationError.cs b/ApiDocs.Validation/Error/ValidationError.cs index 33371d4..e8db096 100644 --- a/ApiDocs.Validation/Error/ValidationError.cs +++ b/ApiDocs.Validation/Error/ValidationError.cs @@ -115,7 +115,8 @@ public enum ValidationErrorCode RequiredScopesMissing, AnnotationParserException, DuplicateMethodIdentifier, - ContentFormatException + ContentFormatException, + NoMatchingMethods } public class ValidationError @@ -172,14 +173,14 @@ public string ErrorText get { StringBuilder sb = new StringBuilder(); - if (this.IsWarning) - { - sb.Append("Warning: "); - } - else if (this.IsError) - { - sb.Append("Error: "); - } + //if (this.IsWarning) + //{ + // sb.Append("Warning: "); + //} + //else if (this.IsError) + //{ + // sb.Append("Error: "); + //} if (!string.IsNullOrEmpty(this.Source)) { diff --git a/ApiDocs.Validation/Error/ValidationWarning.cs b/ApiDocs.Validation/Error/ValidationWarning.cs index 92da7e6..08a83f0 100644 --- a/ApiDocs.Validation/Error/ValidationWarning.cs +++ b/ApiDocs.Validation/Error/ValidationWarning.cs @@ -42,16 +42,19 @@ public ValidationWarning(ValidationErrorCode code, string source, string format, public class UndocumentedPropertyWarning : ValidationWarning { - public UndocumentedPropertyWarning(string source, string propertyName, ParameterDataType propertyType, string resourceName) - : base(ValidationErrorCode.AdditionalPropertyDetected, source, "Undocumented property '{0}' [{1}] was not expected on resource {2}.", propertyName, propertyType, resourceName) + public UndocumentedPropertyWarning(string source, string propertyName, ParameterDataType propertyType, string resourceName, string location = "") + : base(ValidationErrorCode.AdditionalPropertyDetected, source, $"Undocumented property '{propertyName}' [{propertyType}] was not expected on resource {resourceName}. {location}") { this.PropertyName = propertyName; this.PropertyType = propertyType; this.ResourceName = resourceName; + this.Location = location; } public string PropertyName { get; private set; } public ParameterDataType PropertyType { get; private set; } public string ResourceName { get; private set; } + public string Location { get; private set; } } + } diff --git a/ApiDocs.Validation/GitIntegration/GitHelper.cs b/ApiDocs.Validation/GitIntegration/GitHelper.cs new file mode 100644 index 0000000..c2401a6 --- /dev/null +++ b/ApiDocs.Validation/GitIntegration/GitHelper.cs @@ -0,0 +1,128 @@ +/* + * Markdown Scanner + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the ""Software""), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +namespace ApiDocs.Validation +{ + + using System; + using System.IO; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using System.Diagnostics; + + /// + /// Provides a wrapper around processes involving the GIT app + /// + class GitHelper + { + + private string GitExecutablePath { get; set; } + private string RepoDirectoryPath { get; set; } + public GitHelper(string pathToGitExecutable, string repoDirectoryPath) + { + if (!File.Exists(pathToGitExecutable)) + throw new ArgumentException("pathToGit did not specify a path to the GIT executable."); + this.GitExecutablePath = pathToGitExecutable; + + if (!Directory.Exists(repoDirectoryPath)) + throw new ArgumentException("repoDirectoryPath does not exist."); + this.RepoDirectoryPath = repoDirectoryPath; + } + + /// + /// Uses GIT to return a list of files modified in a PR request + /// + /// + /// + public string[] FilesChangedFromBranch(string originalBranch) + { + var baseFetchHeadIdentifier = RunGitCommand($"merge-base HEAD {originalBranch}").TrimEnd(); + var changedFiles = RunGitCommand($"diff --name-only HEAD {baseFetchHeadIdentifier}"); + + var repoChanges = changedFiles.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + var prefix = PrefixForWorkingPath(); + if (string.IsNullOrEmpty(prefix)) + return repoChanges; + + // Remove the prefix for the local working directory, if one exists. + return (from f in repoChanges + where f.StartsWith(prefix) + select f.Substring(prefix.Length)).ToArray(); + } + + public string PrefixForWorkingPath() + { + return RunGitCommand("rev-parse --show-prefix").TrimEnd(); + } + + public static string FindGitLocation() + { + if (System.Environment.OSVersion.Platform == PlatformID.Win32NT) + { + var sys32Folder = Environment.GetFolderPath(Environment.SpecialFolder.System); + var wherePath = Path.Combine(sys32Folder, "where.exe"); + var gitPath = RunCommand(wherePath, "git.exe"); + if (!string.IsNullOrEmpty(gitPath)) + return gitPath.TrimEnd(); + } + + return null; + } + + + private static string RunCommand(string executable, string arguments, string workingDirectory = null) + { + ProcessStartInfo parameters = new ProcessStartInfo(); + parameters.CreateNoWindow = true; + parameters.UseShellExecute = false; + parameters.RedirectStandardError = true; + parameters.RedirectStandardOutput = true; + parameters.FileName = executable; + parameters.Arguments = arguments; + if (null != workingDirectory) + parameters.WorkingDirectory = workingDirectory; + + + var p = Process.Start(parameters); + + StringBuilder sb = new StringBuilder(); + string currentLine = null; + while ((currentLine = p.StandardOutput.ReadLine()) != null) + { + sb.AppendLine(currentLine); + } + + return sb.ToString(); + + } + + private string RunGitCommand(string arguments) + { + return RunCommand(this.GitExecutablePath, arguments, this.RepoDirectoryPath); + } + } +} diff --git a/ApiDocs.Validation/Http/HttpParser.cs b/ApiDocs.Validation/Http/HttpParser.cs index 07c2a1f..6d0f69a 100644 --- a/ApiDocs.Validation/Http/HttpParser.cs +++ b/ApiDocs.Validation/Http/HttpParser.cs @@ -32,7 +32,20 @@ namespace ApiDocs.Validation.Http public class HttpParser { - + + private static HttpParser defaultParser = null; + + public static HttpParser Default + { + get + { + if (null == defaultParser) + { + defaultParser = new HttpParser(); + } + return defaultParser; + } + } /// /// Converts a raw HTTP request into an HttpWebRequest instance. diff --git a/ApiDocs.Validation/Logger/AppveyorLogger.cs b/ApiDocs.Validation/Logger/AppveyorLogger.cs new file mode 100644 index 0000000..895b8aa --- /dev/null +++ b/ApiDocs.Validation/Logger/AppveyorLogger.cs @@ -0,0 +1,17 @@ +using ApiDocs.Validation.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ApiDocs.Validation.Logger +{ + //internal class AppveyorLogger : OutputDelegate + //{ + // public AppveyorLogger(ReportingEngineParameters parameters) : base(parameters) + // { + + // } + //} +} diff --git a/ApiDocs.Validation/Logger/ConsoleLogger.cs b/ApiDocs.Validation/Logger/ConsoleLogger.cs new file mode 100644 index 0000000..0f527e0 --- /dev/null +++ b/ApiDocs.Validation/Logger/ConsoleLogger.cs @@ -0,0 +1,75 @@ +using ApiDocs.Validation.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ApiDocs.Validation.Error; +using System.IO; + +namespace ApiDocs.Validation.Logger +{ + internal class ConsoleLogger : OutputDelegate + { + public ConsoleLogger(ReportingEngineParameters report, SeverityParameters severity) : base(report, severity) + { + + } + public override Task StartTestAsync(TestEngine.DocTest test) + { + if (Report.Level == LogLevel.Verbose) + { + Console.WriteLine($"Starting test: {test.Name}"); + } + + return Task.FromResult(false); + } + + public override async Task ReportTestCompleteAsync(TestEngine.DocTest test, ValidationError[] messages) + { + using (StringWriter writer = new StringWriter()) + { + bool hasWrittenHeader = false; + foreach (var message in messages) + { + var level = TestEngine.LevelForMessage(message, Severity); + // Skip messages unless the output is verbose + if (level == MessageLevel.Message && Report.Level != LogLevel.Verbose) + { + continue; + } + // Skip non-errors if the output is ErrorsOnly + if (level != MessageLevel.Error && Report.Level == LogLevel.ErrorsOnly) + { + continue; + } + + if (!hasWrittenHeader) + { + await writer.WriteLineAsync($"Results for: '{test.Name}'"); + hasWrittenHeader = true; + } + + await WriteValidationErrorAsync(writer, " ", level, message); + } + + if (Report.Level != LogLevel.ErrorsOnly || hasWrittenHeader) + { + await WriteIndentedTextAsync(writer, string.Empty, $"Test '{test.Name}' result: {test.Result} in {test.Duration.TotalSeconds} seconds"); + } + string output = writer.ToString(); + if (output.Any()) + { + Console.WriteLine(writer.ToString()); + } + } + } + + public override Task CloseAsync(TestEngine engine) + { + double percent = 100 * ((double)engine.TestsPassed / (double)engine.TestsPerformed); + Console.WriteLine($"Overall result: {engine.OverallResult}. {engine.TestsPassed} tests of {engine.TestsPerformed} passed ({percent:n1}%)"); + return Task.FromResult(false); + } + } +} diff --git a/ApiDocs.Validation/HttpLog/HttpLogGenerator.cs b/ApiDocs.Validation/Logger/HttpLogGenerator.cs similarity index 100% rename from ApiDocs.Validation/HttpLog/HttpLogGenerator.cs rename to ApiDocs.Validation/Logger/HttpLogGenerator.cs diff --git a/ApiDocs.Validation/Logger/HttpTracerLogger.cs b/ApiDocs.Validation/Logger/HttpTracerLogger.cs new file mode 100644 index 0000000..fa09aff --- /dev/null +++ b/ApiDocs.Validation/Logger/HttpTracerLogger.cs @@ -0,0 +1,17 @@ +using ApiDocs.Validation.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ApiDocs.Validation.Logger +{ + //internal class HttpTracerLogger : OutputDelegate + //{ + // public HttpTracerLogger(ReportingEngineParameters parameters) : base(parameters) + // { + + // } + //} +} diff --git a/ApiDocs.Validation/Logger/MulticastOutputDelegate.cs b/ApiDocs.Validation/Logger/MulticastOutputDelegate.cs new file mode 100644 index 0000000..9c39384 --- /dev/null +++ b/ApiDocs.Validation/Logger/MulticastOutputDelegate.cs @@ -0,0 +1,66 @@ +using ApiDocs.Validation.Error; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ApiDocs.Validation.Logger +{ + internal class MulticastOutputDelegate : IOutputDelegate + { + private IOutputDelegate[] delegates; + + public MulticastOutputDelegate(IEnumerable delegates) + { + this.delegates = delegates.ToArray(); + } + + + public Task ReportTestCompleteAsync(TestEngine.DocTest test, ValidationError[] messages) + { + // RecordUndocumentedProperties(messages); + + return PerformActionAsync(async output => + { + await output.ReportTestCompleteAsync(test, messages); + }); + } + + public Task StartTestAsync(TestEngine.DocTest test) + { + return PerformActionAsync(async output => + { + await output.StartTestAsync(test); + }); + } + + public Task CloseAsync(TestEngine engine) + { + return PerformActionAsync(async output => + { + await output.CloseAsync(engine); + }); + } + + + private async Task PerformActionAsync(Func action) + { + await ForEachAsync(delegates, 4, action); + } + + private static Task ForEachAsync(IEnumerable source, int dop, Func body) + { + return Task.WhenAll( + from partition in Partitioner.Create(source).GetPartitions(dop) + select Task.Run(async delegate { + using (partition) + while (partition.MoveNext()) + await body(partition.Current); + })); + } + + + } +} diff --git a/ApiDocs.Validation/Logger/OutputDelegate.cs b/ApiDocs.Validation/Logger/OutputDelegate.cs new file mode 100644 index 0000000..68b662e --- /dev/null +++ b/ApiDocs.Validation/Logger/OutputDelegate.cs @@ -0,0 +1,56 @@ +using ApiDocs.Validation.Config; +using ApiDocs.Validation.Error; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ApiDocs.Validation.Logger +{ + + internal interface IOutputDelegate + { + Task ReportTestCompleteAsync(TestEngine.DocTest docTest, ValidationError[] messages); + Task StartTestAsync(TestEngine.DocTest test); + Task CloseAsync(TestEngine engine); + } + + + abstract class OutputDelegate : IOutputDelegate + { + protected ReportingEngineParameters Report { get; private set; } + protected SeverityParameters Severity { get; private set; } + + protected OutputDelegate(ReportingEngineParameters report, SeverityParameters severity) + { + Report = report; + Severity = severity; + } + + public abstract Task StartTestAsync(TestEngine.DocTest test); + + public abstract Task ReportTestCompleteAsync(TestEngine.DocTest test, ValidationError[] messages); + + public abstract Task CloseAsync(TestEngine engine); + + protected virtual async Task WriteValidationErrorAsync(System.IO.TextWriter writer, string indent, MessageLevel level, ValidationError message) + { + string prefix = (level == MessageLevel.Error) ? "Error: " : (level == MessageLevel.Warning) ? "Warning: " : ""; + await WriteIndentedTextAsync(writer, indent, String.Concat(prefix, message.ErrorText)); + } + + protected virtual async Task WriteIndentedTextAsync(TextWriter writer, string indent, string text) + { + using (StringReader reader = new StringReader(text)) + { + string line; + while ((line = reader.ReadLine()) != null) + { + await writer.WriteLineAsync(string.Concat(indent, line)); + } + } + } + } +} diff --git a/ApiDocs.Validation/Logger/TestEngine.cs b/ApiDocs.Validation/Logger/TestEngine.cs new file mode 100644 index 0000000..d6320f9 --- /dev/null +++ b/ApiDocs.Validation/Logger/TestEngine.cs @@ -0,0 +1,236 @@ +using ApiDocs.Validation.Error; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ApiDocs.Validation.Config; + +namespace ApiDocs.Validation.Logger +{ + /// + /// Handles routing log output and test scenarios to the approriate destinations based + /// on configuration parameters + /// + public class TestEngine + { + private string DocSetPath { get; set; } + private IOutputDelegate Output { get; set; } + public TestResult OverallResult { get; private set; } + private ApiDocsParameters Parameters { get; set; } + public int TestsPerformed { get; private set; } + public int TestsPassed { get; private set; } + public int TestsFailed { get; private set; } + + public TestEngine(Config.ApiDocsParameters parameters, string docSetPath) + { + DocSetPath = docSetPath; + Parameters = parameters; + + var outputs = CreateOutputs(parameters.Reporting, parameters.Severity); + Output = new MulticastOutputDelegate(outputs); + + OverallResult = TestResult.Running; + + } + + /// + /// Create a set of output delegates based on the parameters + /// + private IEnumerable CreateOutputs(ReportingParameters reports, SeverityParameters severity) + { + severity = severity ?? new SeverityParameters(); + + List outputs = new List(); + //if (reports?.Appveyor != null) + //{ + // outputs.Add(new AppveyorLogger(reports.Appveyor, severity)); + //} + if (reports?.Console != null) + { + outputs.Add(new ConsoleLogger(reports.Console, severity)); + } + //if (reports?.HttpTracer != null) + //{ + // outputs.Add(new HttpTracerLogger(reports.HttpTracer, severity)); + //} + //if (reports?.TextFile != null) + //{ + // outputs.Add(new TextFileLogger(reports.TextFile, severity)); + //} + return outputs; + } + + /// + /// Start logging information about a test. This frequently includes starting + /// a timer for the duration of the test as well. + /// + public async Task StartTestAsync(string name) + { + TestsPerformed += 1; + + var test = new DocTest(this, name); + await Output.StartTestAsync(test); + return test; + } + + public async Task CompleteAsync() + { + await Output.CloseAsync(this); + } + + /// + /// Represnts the output from an individual test in the system + /// + public class DocTest + { + public string Name { get; private set; } + public TimeSpan Duration { get; set; } + public TestResult Result { get; set; } + private DateTimeOffset StartDateTime { get; set; } + private TestEngine Parent { get; set; } + private List Messages { get; set; } + + internal DocTest(TestEngine parent, string testName) + { + this.Parent = parent; + this.Name = testName; + this.StartDateTime = DateTimeOffset.UtcNow; + this.Messages = new List(); + this.Duration = TimeSpan.MaxValue; + this.Result = TestResult.NotStarted; + } + + public void LogMessage(ValidationError message) + { + if (message != null) + { + Messages.Add(message); + } + } + + public void LogMessages(IEnumerable messages) + { + if (messages != null) + { + Messages.AddRange(messages); + } + } + + public async Task CompleteAsync(ValidationError error, TestResult? overrideResult = null) + { + return await CompleteAsync(new ValidationError[] { error }, overrideResult); + } + + public async Task CompleteAsync(IEnumerable errors, TestResult? overrideResult = null) + { + Duration = DateTimeOffset.UtcNow.Subtract(this.StartDateTime); + LogMessages(errors); + + Result = DetermineTestResult(); + await Parent.Output.ReportTestCompleteAsync(this, Messages.ToArray()); + + if (Result == TestResult.Passed || Result == TestResult.PassedWithWarnings) + { + Parent.TestsPassed += 1; + } + else + { + Parent.TestsFailed += 1; + } + + Parent.OverallResult = (new TestResult[] { Result, Parent.OverallResult }).Min(); + return Result; + } + + private TestResult DetermineTestResult() + { + var severity = Parent.Parameters.Severity; + var query = from m in Messages + select ResultFromMessage(m, Parent.Parameters.Severity); + if (query.Any()) + { + return query.Min(); + } + else + { + return TestResult.Passed; + } + } + + internal static TestResult ResultFromMessage(ValidationError message, SeverityParameters severity) + { + if (message.IsError && ( severity.Errors == SeverityLevel.Critical || severity.Errors == SeverityLevel.Default )) + { + return TestResult.Failed; + } + else if (message.IsError && severity.Errors == SeverityLevel.Warning) + { + return TestResult.PassedWithWarnings; + } + else if (message.IsWarning && severity.Warnings == SeverityLevel.Critical) + { + return TestResult.Failed; + } + else if (message.IsWarning && (severity.Warnings == SeverityLevel.Warning || severity.Warnings == SeverityLevel.Default )) + { + return TestResult.PassedWithWarnings; + } + else + { + return TestResult.Passed; + } + } + + + } + + /// + /// Determine the message level for a message, based on it's own definition and the severity parameters + /// + /// + /// + /// + internal static MessageLevel LevelForMessage(ValidationError message, SeverityParameters severity) + { + if (message.IsError && (severity.Errors == SeverityLevel.Critical || severity.Errors == SeverityLevel.Default)) + { + return MessageLevel.Error; + } + else if (message.IsError && severity.Errors == SeverityLevel.Warning) + { + return MessageLevel.Warning; + } + else if (message.IsWarning && severity.Warnings == SeverityLevel.Critical) + { + return MessageLevel.Error; + } + else if (message.IsWarning && (severity.Warnings == SeverityLevel.Warning || severity.Warnings == SeverityLevel.Default)) + { + return MessageLevel.Warning; + } + + return MessageLevel.Message; + } + + } + + public enum MessageLevel + { + Message = 0, + Warning, + Error + } + + public enum TestResult + { + NotStarted = 0, + NothingToTest = 50, + Failed = 100, + PassedWithWarnings = 200, + Passed = 300, + Running = 1000 + + } + +} diff --git a/ApiDocs.Validation/Logger/TextFileLogger.cs b/ApiDocs.Validation/Logger/TextFileLogger.cs new file mode 100644 index 0000000..03872dd --- /dev/null +++ b/ApiDocs.Validation/Logger/TextFileLogger.cs @@ -0,0 +1,19 @@ +using ApiDocs.Validation.Config; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ApiDocs.Validation.Logger +{ + //internal class TextFileLogger : OutputDelegate + //{ + // public TextFileLogger(ReportingEngineParameters report, SeverityParameters severity) : base(parameters, severity) + // { + + // } + + + //} +} diff --git a/ApiDocs.Validation/MethodDefinition.cs b/ApiDocs.Validation/MethodDefinition.cs index 6d700ce..005841e 100644 --- a/ApiDocs.Validation/MethodDefinition.cs +++ b/ApiDocs.Validation/MethodDefinition.cs @@ -130,7 +130,7 @@ public void AddExpectedResponse(string rawResponse, CodeBlockAnnotation annotati { if (this.ExpectedResponse != null) { - throw new InvalidOperationException("An expected response was already added to this request."); + throw new MethodDuplicationException("An expected response was already added to this request."); } this.ExpectedResponse = rawResponse; diff --git a/ApiDocs.Validation/MethodException.cs b/ApiDocs.Validation/MethodException.cs new file mode 100644 index 0000000..8a69115 --- /dev/null +++ b/ApiDocs.Validation/MethodException.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ApiDocs.Validation +{ + public class MethodDuplicationException : Exception + { + public MethodDuplicationException(string message) : base(message) + { + + } + } +} diff --git a/ApiDocs.Validation/ValidationProcessor.cs b/ApiDocs.Validation/ValidationProcessor.cs new file mode 100644 index 0000000..425a437 --- /dev/null +++ b/ApiDocs.Validation/ValidationProcessor.cs @@ -0,0 +1,250 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ApiDocs.Validation.Config; +using ApiDocs.Validation.Error; +using ApiDocs.Validation.Logger; + +namespace ApiDocs.Validation +{ + /// + /// Wraps the various validation methods implemented into configuration + simple functions + /// + public static class ValidationProcessor + { + /// + /// Performs the check-links operation, which validates that there are no broken links within the DocSet. + /// + public static async Task CheckLinksAsync(ApiDocsParameters parameters, DocSet docs, TestEngine tester = null) + { + if (null == parameters) throw new ArgumentNullException("parameters"); + if (null == docs) throw new ArgumentNullException("docs"); + + var filesOfInterest = ComputeFilesOfInterest(parameters, docs); + tester = tester ?? new TestEngine(parameters, docs.SourceFolderPath); + + await ValidateLinks(parameters.CheckLinksParameters, docs, filesOfInterest, tester); + + return tester.OverallResult; + } + + /// + /// Perform a series of tests on the DocSet provided based on the parameters specified. + /// These tests will evaluate the content of the documentation files to ensure correctness. + /// + public static async Task CheckDocsAsync(ApiDocsParameters parameters, DocSet docs, TestEngine tester = null) + { + if (null == parameters) throw new ArgumentNullException("parameters"); + if (null == docs) throw new ArgumentNullException("docs"); + + var checkDocs = parameters.CheckDocsParameters ?? new CheckDocsActionParameters(); + + var filesOfInterest = ComputeFilesOfInterest(parameters, docs); + tester = tester ?? new TestEngine(parameters, docs.SourceFolderPath); + + // Look for structure errors in the docs + if (checkDocs.ValidateStructure) + { + await ValidateDocumentStructure(checkDocs, docs, filesOfInterest, tester); + } + + // Check for errors in request/response pairs (methods) + if (checkDocs.ValidateMethods) + { + await CheckMethodsAsync(checkDocs, docs, filesOfInterest, tester); + } + + // Check for errors in code examples + if (checkDocs.ValidateExamples) + { + await CheckExamplesAsync(checkDocs, docs, filesOfInterest, tester); + } + + return tester.OverallResult; + } + + /// + /// Validate that links within the documentation are correct. + /// + private static async Task ValidateLinks(CheckLinksActionParameters parameters, DocSet docs, IEnumerable filesOfInterest, TestEngine tester) + { + parameters = parameters ?? new CheckLinksActionParameters(); + + var test = await tester.StartTestAsync("Validate links"); + IEnumerable files = (filesOfInterest.Any()) ? filesOfInterest : docs.Files; + + List detectedErrors = new List(); + foreach(var file in files) + { + ValidationError[] errors; + file.ValidateNoBrokenLinks(true, out errors, parameters.LinksAreCaseSensitive); + detectedErrors.AddRange(errors); + } + await test.CompleteAsync(detectedErrors); + } + + /// + /// Validate that the structure of the document is correct. This includes headings, table columns, and other structural elements + /// + private static async Task ValidateDocumentStructure(CheckDocsActionParameters parameters, DocSet docs, IEnumerable filesOfInterest, TestEngine tester) + { + var test = await tester.StartTestAsync("Validate document structure"); + IEnumerable files = (filesOfInterest.Any()) ? filesOfInterest : docs.Files; + + var detectedErrors = new List(); + foreach (var file in files) + { + var errors = file.CheckDocumentStructure(); + detectedErrors.AddRange(errors); + } + + await test.CompleteAsync(detectedErrors); + } + + /// + /// Validate that code examples within the documentation are correct. + /// + private static async Task CheckExamplesAsync(CheckDocsActionParameters parameters, DocSet docs, IEnumerable filesOfInterest, TestEngine tester) + { + IEnumerable files = (filesOfInterest.Any()) ? filesOfInterest : docs.Files; + foreach (var file in files) + { + if (!file.Examples.Any()) + { + continue; + } + + foreach (var example in file.Examples) + { + if (example.Metadata == null) + { + continue; + } + + var test = await tester.StartTestAsync($"Example: {example.Metadata.MethodName} in {file.DisplayName}"); + ValidationError[] errors; + switch (example.Language) + { + case CodeLanguage.Json: + { + docs.ResourceCollection.ValidateJsonExample(example.Metadata, example.SourceExample, out errors, new Json.ValidationOptions { RelaxedStringValidation = parameters.RelaxStringValidation }); + break; + } + default: + { + errors = new ValidationError[] { new ValidationWarning(ValidationErrorCode.UnsupportedLanguage, file.DisplayName, $"Example {example.Metadata.MethodName} was skipped because {example.Language} is not supported.") }; + break; + } + } + await test.CompleteAsync(errors); + } + } + } + + /// + /// Validate that request/response pairs (methods) within the documentation are correct. + /// + private static async Task CheckMethodsAsync(CheckDocsActionParameters parameters, DocSet docs, IEnumerable filesOfInterest, TestEngine tester) + { + IEnumerable files = (filesOfInterest.Any()) ? filesOfInterest : docs.Files; + var methods = GetMethodsForTesting(files, parameters); + + if (!methods.Any()) + { + var test = await tester.StartTestAsync("check-methods"); + await test.CompleteAsync(new ValidationError(ValidationErrorCode.NoMatchingMethods, null, $"No methods matches the provided filters."), TestResult.Failed); + return; + } + + foreach(var method in methods) + { + var test = await tester.StartTestAsync($"check-method: {method.Identifier} in {method.SourceFile.DisplayName}"); + + // Make sure the method has a response + if (string.IsNullOrEmpty(method.ExpectedResponse)) + { + await test.CompleteAsync(new ValidationError(ValidationErrorCode.RequestWasEmptyOrNull, method.SourceFile.DisplayName, $"Response was null or empty."), TestResult.Failed); + continue; + } + + var parser = Http.HttpParser.Default; + ValidationError[] errors; + try + { + var expectedResponse = parser.ParseHttpResponse(method.ExpectedResponse); + method.ValidateResponse(expectedResponse, null, null, out errors, new Json.ValidationOptions { RelaxedStringValidation = parameters.RelaxStringValidation }); + } + catch (Exception ex) + { + errors = new ValidationError[] { new ValidationError(ValidationErrorCode.ExceptionWhileValidatingMethod, method.SourceFile.DisplayName, ex.Message) }; + } + + await test.CompleteAsync(errors); + } + } + + /// + /// Returns an iterator for MethodDefintion instances in a collection of DocFiles that match a specified set of filters. + /// + private static IEnumerable GetMethodsForTesting(IEnumerable files, CheckDocsActionParameters options) + { + if (!string.IsNullOrEmpty(options.MethodFilter)) + { + return (from m in GetMethodsInFiles(files) + where m.Identifier.IsWildcardMatch(options.MethodFilter) + select m); + } + else if (!string.IsNullOrEmpty(options.FilenameFilter)) + { + var matchingFiles = (from f in files where f.DisplayName.IsWildcardMatch(options.FilenameFilter) select f); + return GetMethodsInFiles(matchingFiles); + } + else + { + return GetMethodsInFiles(files); + } + } + + /// + /// Iterate through the methods in a collection of files + /// + /// + /// + private static IEnumerable GetMethodsInFiles(IEnumerable files) + { + foreach(var file in files) + { + foreach(var method in file.Requests) + { + yield return method; + } + } + } + + /// + /// Uses the parameters and docs to figure out a set of interesting files within the docs. + /// + private static IEnumerable ComputeFilesOfInterest(ApiDocsParameters parameters, DocSet docs) + { + // If this is a pull request, and we have a target branch to compare with, then find the files that are different + if (!string.IsNullOrEmpty(parameters.PullRequests?.PullRequestDetector) && !string.IsNullOrEmpty(parameters.PullRequests?.TargetBranch)) + { + GitHelper helper = new GitHelper(parameters.GitExecutablePath, docs.SourceFolderPath); + var pathsOfInterestingFiles = helper.FilesChangedFromBranch(parameters.PullRequests.TargetBranch); + + foreach(var path in pathsOfInterestingFiles) + { + var file = docs.LookupFileForPath(path); + if (null != file) + { + yield return file; + } + } + } + yield break; + } + + } +} diff --git a/ApiDocs.Validation/WildcardExtensions.cs b/ApiDocs.Validation/WildcardExtensions.cs new file mode 100644 index 0000000..464e66e --- /dev/null +++ b/ApiDocs.Validation/WildcardExtensions.cs @@ -0,0 +1,51 @@ +/* + * Markdown Scanner + * Copyright (c) Microsoft Corporation + * All rights reserved. + * + * MIT License + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the ""Software""), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + + +namespace ApiDocs.Validation +{ + using System.Text.RegularExpressions; + + internal static class WildcardExtensions + { + /// + /// Convert a wildcard string pattern to a RegEx. + /// + /// + /// + private static string WildcardToRegex(string pattern) + { + return "^" + Regex.Escape(pattern) + .Replace(@"\*", ".*") + .Replace(@"\?", ".") + + "$"; + } + + public static bool IsWildcardMatch(this string source, string pattern) + { + return Regex.IsMatch(source, WildcardToRegex(pattern)); + } + } +} diff --git a/MarkdownScanner.sln b/MarkdownScanner.sln index 482bb15..c583d95 100644 --- a/MarkdownScanner.sln +++ b/MarkdownScanner.sln @@ -5,6 +5,7 @@ VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Content", "Content", "{464F5D09-2636-459F-96C9-C968FF4CA7B5}" ProjectSection(SolutionItems) = preProject + config-file-example.json = config-file-example.json license.txt = license.txt OpenSourceNotes.md = OpenSourceNotes.md readme.md = readme.md diff --git a/config-file-example.json b/config-file-example.json new file mode 100644 index 0000000..40e482c --- /dev/null +++ b/config-file-example.json @@ -0,0 +1,86 @@ +{ + /* define the documentation sets contains in this repo */ + "sets": { + "all-content": { + "path": "/", + "actions": { + "check-links": { } + } + }, + "v1.0-reference": { + "path": "/api-reference/v1.0", + "actions": { + "check-docs": { }, + "check-service": { + "accounts": [ "MSGraphProd", "MSGraphBeta", "MSGraphPPE" ] + }, + "check-metadata": { + "schema-urls": [ "https://graph.microsoft.com/v1.0/$metadata" ] + } + } + }, + "beta-reference": { + "path": "/api-reference/beta", + "actions": { + "check-docs": {} + } + }, + "publish-docs": { + "path": "/", + "actions": { + "publish": { + "output": "/path/to/output", + "format": "mustache", + "template": { + "path": "/path/to/template", + "topics": "template.htm" + }, + "file-extension": ".htm", + "line-ending": "default | windows | unix | mac", + "toc": { + "format": "json", + "path": "/path/to/toc-output-file" + }, + "allow-unsafe-html": false + } + } + } + }, + + /* default-parameters applied to all tests. */ + "default-parameters": { + "reporting": { + "console": { + "log-level": "default | none | errorsOnly | verbose" + }, + "appveyor": { + "log-level": "default | none | errorsOnly | verbose", + "url": "$env:APPVEYOR_API_URL" + }, + "text-file": { + "log-level": "default | none | errorsOnly | verbose", + "path": "/path/to/file" + }, + "fiddler": { + "path": "/path/to/file" + } + }, + "severity": { + "errors": "default | critical | warn | ignore", + "warnings": "default | critical | warn | ignore", + "messages": "default | critical | warn | ignore" + }, + "pull-requests": { + "pull-request-indicator": "$env:APPVEYOR_PULL_REQUEST_NUMBER", /* when this value is non-null, we're doing a pull request */ + "target-branch": "$env:APPVEYOR_REPO_BRANCH" /* could also just be master */ + }, + "check-docs": { + "relax-string-validation": true + }, + "check-links": { + "case-sensitive": true + }, + "git-path": "/path/to/git.exe" + } +} +