From c470e0db676ed678190c5cb359883921b12e7c9d Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Thu, 29 Feb 2024 15:52:56 -0500 Subject: [PATCH 01/17] - adds migrate command handler --- specs/cli/config-migrate.md | 6 +-- src/kiota/Handlers/Config/MigrateHandler.cs | 47 +++++++++++++++++++++ src/kiota/KiotaClientCommands.cs | 2 +- src/kiota/KiotaConfigCommands.cs | 23 +++++++++- 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 src/kiota/Handlers/Config/MigrateHandler.cs diff --git a/specs/cli/config-migrate.md b/specs/cli/config-migrate.md index 0992e3dcd8..86b3fa2bb8 100644 --- a/specs/cli/config-migrate.md +++ b/specs/cli/config-migrate.md @@ -8,8 +8,8 @@ In the case where conflicting API client names would be migrated, the command wi | Parameters | Required | Example | Description | | -- | -- | -- | -- | -| `--lock-location \| --ll` | No | ./output/pythonClient/kiota-lock.json | Location of the `kiota-lock.json` file. If not specified, all `kiota-lock.json` files within in the current directory tree will be used. | -| `--client-name \| --cn` | No | GraphClient | Used with `--lock-location`, it would allow to specify a name for the API client. Else, name is auto-generated as a concatenation of the `language` and `clientClassName`. | +| `--lock-directory \| --ld` | No | ./output/pythonClient/ | Relative path to the directory containing the `kiota-lock.json` file. If not specified, all `kiota-lock.json` files within in the current directory tree will be used. | +| `--client-name \| --cn` | No | GraphClient | Used with `--lock-directory`, it would allow to specify a name for the API client. Else, name is auto-generated as a concatenation of the `language` and `clientClassName`. | ## Using `kiota config migrate` @@ -181,7 +181,7 @@ Assuming the following folder structure: ``` ```bash -kiota config migrate --lock-location ./generated/graph/csharp/kiota-lock.json --client-name GraphClient +kiota config migrate --lock-directory ./generated/graph/csharp --client-name GraphClient ``` _The resulting `kiota-config.json` file will look like this:_ diff --git a/src/kiota/Handlers/Config/MigrateHandler.cs b/src/kiota/Handlers/Config/MigrateHandler.cs new file mode 100644 index 0000000000..4c94400085 --- /dev/null +++ b/src/kiota/Handlers/Config/MigrateHandler.cs @@ -0,0 +1,47 @@ +using System; +using System.CommandLine; +using System.CommandLine.Invocation; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Kiota.Builder.WorkspaceManagement; +using Microsoft.Extensions.Logging; + +namespace kiota.Handlers.Config; + +internal class MigrateHandler : BaseKiotaCommandHandler +{ + public required Option LockDirectoryOption + { + get; + init; + } + public required Option ClassOption + { + get; init; + } + + public async override Task InvokeAsync(InvocationContext context) + { + string lockDirectory = context.ParseResult.GetValueForOption(LockDirectoryOption) ?? Directory.GetCurrentDirectory(); + string className = context.ParseResult.GetValueForOption(ClassOption) ?? string.Empty; + CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; + AssignIfNotNullOrEmpty(className, (c, s) => c.ClientClassName = s); + var workspaceStorageService = new WorkspaceConfigurationStorageService(Directory.GetCurrentDirectory()); + var (loggerFactory, logger) = GetLoggerAndFactory(context, Configuration.Generation.OutputPath); + using (loggerFactory) + { + try + { + await Task.Delay(0).ConfigureAwait(false); + // await workspaceStorageService.MigrateFromLockFileAsync(cancellationToken).ConfigureAwait(false); + return 0; + } + catch (Exception ex) + { + logger.LogCritical(ex, "error migrating the workspace configuration"); + return 1; + } + } + } +} diff --git a/src/kiota/KiotaClientCommands.cs b/src/kiota/KiotaClientCommands.cs index aac0085d29..97dba4e76b 100644 --- a/src/kiota/KiotaClientCommands.cs +++ b/src/kiota/KiotaClientCommands.cs @@ -20,7 +20,7 @@ private static Option GetSkipGenerationOption() skipGeneration.AddAlias("--sg"); return skipGeneration; } - private static Option GetClientNameOption() + internal static Option GetClientNameOption() { var clientName = new Option("--client-name", "The name of the client to manage") { diff --git a/src/kiota/KiotaConfigCommands.cs b/src/kiota/KiotaConfigCommands.cs index a69cdddcf3..abb272ea8d 100644 --- a/src/kiota/KiotaConfigCommands.cs +++ b/src/kiota/KiotaConfigCommands.cs @@ -26,8 +26,27 @@ private static Command GetInitCommand() } private static Command GetMigrateCommand() { - var command = new Command("migrate", "Migrates a kiota lock file to a Kiota configuration"); - //TODO map the handler + var logLevelOption = KiotaHost.GetLogLevelOption(); + var lockDirectoryOption = GetLockDirectoryOption(); + var classOption = KiotaClientCommands.GetClientNameOption(); + var command = new Command("migrate", "Migrates a kiota lock file to a Kiota configuration") + { + logLevelOption, + lockDirectoryOption, + classOption, + }; + command.Handler = new MigrateHandler + { + LogLevelOption = logLevelOption, + LockDirectoryOption = lockDirectoryOption, + ClassOption = classOption, + }; return command; } + private static Option GetLockDirectoryOption() + { + var option = new Option("--lock-directory", "The directory containing a kiota-lock.json file"); + option.AddAlias("--ld"); + return option; + } } From 0bd339c50fc785aa2d036c33f13f5eca490941da Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Thu, 29 Feb 2024 16:54:40 -0500 Subject: [PATCH 02/17] - draft implementation of the migration logic Signed-off-by: Vincent Biret --- .../Lock/LockManagementService.cs | 9 ++- .../WorkspaceConfigurationStorageService.cs | 2 + .../WorkspaceManagementService.cs | 67 +++++++++++++++++++ src/kiota/Handlers/Config/MigrateHandler.cs | 24 ++++--- 4 files changed, 92 insertions(+), 10 deletions(-) diff --git a/src/Kiota.Builder/Lock/LockManagementService.cs b/src/Kiota.Builder/Lock/LockManagementService.cs index 10e720835e..db4a17c30d 100644 --- a/src/Kiota.Builder/Lock/LockManagementService.cs +++ b/src/Kiota.Builder/Lock/LockManagementService.cs @@ -15,7 +15,7 @@ namespace Kiota.Builder.Lock; /// public class LockManagementService : ILockManagementService { - private const string LockFileName = "kiota-lock.json"; + internal const string LockFileName = "kiota-lock.json"; /// public IEnumerable GetDirectoriesContainingLockFile(string searchDirectory) { @@ -132,4 +132,11 @@ private static string GetBackupFilePath(string outputPath) var hashedPath = BitConverter.ToString((HashAlgorithm.Value ?? throw new InvalidOperationException("unable to get hash algorithm")).ComputeHash(Encoding.UTF8.GetBytes(outputPath))).Replace("-", string.Empty, StringComparison.OrdinalIgnoreCase); return Path.Combine(Path.GetTempPath(), Constants.TempDirectoryName, "backup", hashedPath, LockFileName); } + public void DeleteLockFile(string directoryPath) + { + ArgumentException.ThrowIfNullOrEmpty(directoryPath); + var lockFilePath = Path.Combine(directoryPath, LockFileName); + if (File.Exists(lockFilePath)) + File.Delete(lockFilePath); + } } diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs index 02cb20ca51..636d1e443e 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs @@ -1,4 +1,6 @@ using System; +using System.Collections; +using System.Collections.Generic; using System.IO; using System.Security.Cryptography; using System.Text; diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index 43b481c2a0..5d7268e836 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Security.Cryptography; using System.Text; using System.Text.Json; @@ -172,4 +173,70 @@ private static string ConvertByteArrayToString(byte[] hash) return sb.ToString(); } + + public async Task> MigrateFromLockFileAsync(string clientName, string lockDirectory, CancellationToken cancellationToken = default) + { + ArgumentException.ThrowIfNullOrEmpty(lockDirectory); + if (!UseKiotaConfig) + throw new InvalidOperationException("Cannot migrate from lock file in kiota config mode"); + if (!Path.IsPathRooted(lockDirectory)) + lockDirectory = Path.Combine(WorkingDirectory, lockDirectory); + if (Path.GetRelativePath(WorkingDirectory, lockDirectory).StartsWith("..", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException("The lock directory must be a subdirectory of the working directory"); + var (wsConfig, apiManifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); + wsConfig ??= new WorkspaceConfiguration(); + apiManifest ??= new ApiManifestDocument("application"); //TODO get the application name + var lockFiles = Directory.GetFiles(lockDirectory, LockManagementService.LockFileName, SearchOption.AllDirectories); + if (lockFiles.Length == 0) + throw new InvalidOperationException("No lock file found in the specified directory"); + var clientNamePassed = !string.IsNullOrEmpty(clientName); + if (lockFiles.Length > 1 && clientNamePassed) + throw new InvalidOperationException("Multiple lock files found in the specified directory and the client name was specified"); + var clientsGenerationConfigurations = new List(); + if (lockFiles.Length == 1) + clientsGenerationConfigurations.Add(await LoadConfigurationFromLockAsync(clientNamePassed ? clientName : string.Empty, lockFiles[0], cancellationToken).ConfigureAwait(false)); + else + clientsGenerationConfigurations.AddRange(await Task.WhenAll(lockFiles.Select(x => LoadConfigurationFromLockAsync(string.Empty, x, cancellationToken))).ConfigureAwait(false)); + var loadedConfigurations = clientsGenerationConfigurations.OfType().ToArray(); + foreach (var configuration in loadedConfigurations) + { + var generationClientConfig = new ApiClientConfiguration(configuration); + generationClientConfig.NormalizePaths(WorkingDirectory); + if (wsConfig.Clients.ContainsKey(configuration.ClientClassName)) + { + Logger.LogError("The client {ClientName} is already present in the configuration", configuration.ClientClassName); + clientsGenerationConfigurations.Remove(configuration); + continue; + } + wsConfig.Clients.Add(configuration.ClientClassName, generationClientConfig); + var inputConfigurationHash = await GetConfigurationHashAsync(generationClientConfig, "migrated").ConfigureAwait(false); + // because it's a migration, we don't want to calculate the exact hash since the description might have changed since the initial generation that created the lock file + apiManifest.ApiDependencies.Add(configuration.ClientClassName, configuration.ToApiDependency(inputConfigurationHash, new()));//TODO get the resolved operations? + //TODO copy the description file + lockManagementService.DeleteLockFile(Path.GetDirectoryName(configuration.OpenAPIFilePath)!); + } + await workspaceConfigurationStorageService.UpdateWorkspaceConfigurationAsync(wsConfig, apiManifest, cancellationToken).ConfigureAwait(false); + return clientsGenerationConfigurations.OfType().Select(static x => x.ClientClassName); + } + private async Task LoadConfigurationFromLockAsync(string clientName, string lockFilePath, CancellationToken cancellationToken) + { + if (Path.GetDirectoryName(lockFilePath) is not string lockFileDirectory) + { + Logger.LogWarning("The lock file {LockFilePath} is not in a directory, it will be skipped", lockFilePath); + return null; + } + var lockInfo = await lockManagementService.GetLockFromDirectoryAsync(lockFileDirectory, cancellationToken).ConfigureAwait(false); + if (lockInfo is null) + { + Logger.LogWarning("The lock file {LockFilePath} is not valid, it will be skipped", lockFilePath); + return null; + } + var generationConfiguration = new GenerationConfiguration(); + lockInfo.UpdateGenerationConfigurationFromLock(generationConfiguration); + if (!string.IsNullOrEmpty(clientName)) + { + generationConfiguration.ClientClassName = clientName; + } + return generationConfiguration; + } } diff --git a/src/kiota/Handlers/Config/MigrateHandler.cs b/src/kiota/Handlers/Config/MigrateHandler.cs index 4c94400085..a1b05ade6f 100644 --- a/src/kiota/Handlers/Config/MigrateHandler.cs +++ b/src/kiota/Handlers/Config/MigrateHandler.cs @@ -2,6 +2,7 @@ using System.CommandLine; using System.CommandLine.Invocation; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Kiota.Builder.WorkspaceManagement; @@ -20,21 +21,26 @@ public required Option ClassOption { get; init; } - - public async override Task InvokeAsync(InvocationContext context) + public override async Task InvokeAsync(InvocationContext context) { - string lockDirectory = context.ParseResult.GetValueForOption(LockDirectoryOption) ?? Directory.GetCurrentDirectory(); - string className = context.ParseResult.GetValueForOption(ClassOption) ?? string.Empty; + var workingDirectory = NormalizeSlashesInPath(Directory.GetCurrentDirectory()); + string lockDirectory = context.ParseResult.GetValueForOption(LockDirectoryOption) ?? workingDirectory; + string clientName = context.ParseResult.GetValueForOption(ClassOption) ?? string.Empty; CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; - AssignIfNotNullOrEmpty(className, (c, s) => c.ClientClassName = s); - var workspaceStorageService = new WorkspaceConfigurationStorageService(Directory.GetCurrentDirectory()); - var (loggerFactory, logger) = GetLoggerAndFactory(context, Configuration.Generation.OutputPath); + lockDirectory = NormalizeSlashesInPath(lockDirectory); + var (loggerFactory, logger) = GetLoggerAndFactory(context, Configuration.Generation.OutputPath); using (loggerFactory) { try { - await Task.Delay(0).ConfigureAwait(false); - // await workspaceStorageService.MigrateFromLockFileAsync(cancellationToken).ConfigureAwait(false); + var workspaceManagementService = new WorkspaceManagementService(logger, true, workingDirectory); + var clientNames = await workspaceManagementService.MigrateFromLockFileAsync(clientName, lockDirectory, cancellationToken).ConfigureAwait(false); + if (!clientNames.Any()) + { + logger.LogWarning("no client configuration was migrated"); + return 1; + } + logger.LogInformation("client configurations migrated successfully: {clientNames}", string.Join(", ", clientNames)); return 0; } catch (Exception ex) From 12c3f948845a5e02a2b97a0a9337e69661308afb Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 09:19:30 -0500 Subject: [PATCH 03/17] - adds the description copy upon migration Signed-off-by: Vincent Biret --- src/Kiota.Builder/DownloadHelper.cs | 80 +++++++++++++++++++ src/Kiota.Builder/KiotaBuilder.cs | 63 +-------------- .../WorkspaceManagementService.cs | 26 +++++- src/kiota/Handlers/Client/RemoveHandler.cs | 2 +- src/kiota/Handlers/Config/MigrateHandler.cs | 2 +- .../WorkspaceManagementServiceTests.cs | 22 ++--- 6 files changed, 120 insertions(+), 75 deletions(-) create mode 100644 src/Kiota.Builder/DownloadHelper.cs diff --git a/src/Kiota.Builder/DownloadHelper.cs b/src/Kiota.Builder/DownloadHelper.cs new file mode 100644 index 0000000000..d2834006ab --- /dev/null +++ b/src/Kiota.Builder/DownloadHelper.cs @@ -0,0 +1,80 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Net.Http; +using System.Security; +using System.Threading; +using System.Threading.Tasks; +using AsyncKeyedLock; +using Kiota.Builder.Caching; +using Kiota.Builder.Configuration; +using Kiota.Builder.Extensions; +using Kiota.Builder.SearchProviders.APIsGuru; +using Kiota.Builder.WorkspaceManagement; +using Microsoft.Extensions.Logging; + +namespace Kiota.Builder; +internal static class DownloadHelper +{ + internal static async Task<(Stream, bool)> LoadStream(string inputPath, HttpClient httpClient, ILogger logger, GenerationConfiguration config, AsyncKeyedLocker localFilesLock, WorkspaceManagementService? workspaceManagementService = default, bool useKiotaConfig = false, CancellationToken cancellationToken = default) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + inputPath = inputPath.Trim(); + + Stream input; + var isDescriptionFromWorkspaceCopy = false; + if (useKiotaConfig && + config.Operation is ClientOperation.Edit or ClientOperation.Add && + workspaceManagementService is not null && + await workspaceManagementService.GetDescriptionCopyAsync(config.ClientClassName, inputPath, cancellationToken).ConfigureAwait(false) is { } descriptionStream) + { + logger.LogInformation("loaded description from the workspace copy"); + input = descriptionStream; + isDescriptionFromWorkspaceCopy = true; + } + else if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + try + { + var cachingProvider = new DocumentCachingProvider(httpClient, logger) + { + ClearCache = config.ClearCache, + }; + var targetUri = APIsGuruSearchProvider.ChangeSourceUrlToGitHub(new Uri(inputPath)); // so updating existing clients doesn't break + var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml"; + input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); + logger.LogInformation("loaded description from remote source"); + } + catch (HttpRequestException ex) + { + throw new InvalidOperationException($"Could not download the file at {inputPath}, reason: {ex.Message}", ex); + } + else + try + { + var inMemoryStream = new MemoryStream(); + using (await localFilesLock.LockAsync(inputPath, cancellationToken).ConfigureAwait(false)) + {// To avoid deadlocking on update with multiple clients for the same local description + using var fileStream = new FileStream(inputPath, FileMode.Open); + await fileStream.CopyToAsync(inMemoryStream, cancellationToken).ConfigureAwait(false); + } + inMemoryStream.Position = 0; + input = inMemoryStream; + logger.LogInformation("loaded description from local source"); + } + catch (Exception ex) when (ex is FileNotFoundException || + ex is PathTooLongException || + ex is DirectoryNotFoundException || + ex is IOException || + ex is UnauthorizedAccessException || + ex is SecurityException || + ex is NotSupportedException) + { + throw new InvalidOperationException($"Could not open the file at {inputPath}, reason: {ex.Message}", ex); + } + stopwatch.Stop(); + logger.LogTrace("{Timestamp}ms: Read OpenAPI file {File}", stopwatch.ElapsedMilliseconds, inputPath); + return (input, isDescriptionFromWorkspaceCopy); + } +} diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index 200458a1e4..9cd0706c7a 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Net.Http; using System.Runtime.CompilerServices; -using System.Security; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -25,7 +24,6 @@ using Kiota.Builder.Manifest; using Kiota.Builder.OpenApiExtensions; using Kiota.Builder.Refiners; -using Kiota.Builder.SearchProviders.APIsGuru; using Kiota.Builder.Validation; using Kiota.Builder.WorkspaceManagement; using Kiota.Builder.Writers; @@ -64,7 +62,7 @@ public KiotaBuilder(ILogger logger, GenerationConfiguration config MaxDegreeOfParallelism = config.MaxDegreeOfParallelism, }; var workingDirectory = Directory.GetCurrentDirectory(); - workspaceManagementService = new WorkspaceManagementService(logger, useKiotaConfig, workingDirectory); + workspaceManagementService = new WorkspaceManagementService(logger, client, useKiotaConfig, workingDirectory); this.useKiotaConfig = useKiotaConfig; } private readonly bool useKiotaConfig; @@ -402,63 +400,8 @@ private void StopLogAndReset(Stopwatch sw, string prefix) private bool isDescriptionFromWorkspaceCopy; private async Task LoadStream(string inputPath, CancellationToken cancellationToken) { - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - inputPath = inputPath.Trim(); - - Stream input; - if (useKiotaConfig && - config.Operation is ClientOperation.Edit or ClientOperation.Add && - await workspaceManagementService.GetDescriptionCopyAsync(config.ClientClassName, inputPath, cancellationToken).ConfigureAwait(false) is { } descriptionStream) - { - logger.LogInformation("loaded description from the workspace copy"); - input = descriptionStream; - isDescriptionFromWorkspaceCopy = true; - } - else if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) - try - { - var cachingProvider = new DocumentCachingProvider(httpClient, logger) - { - ClearCache = config.ClearCache, - }; - var targetUri = APIsGuruSearchProvider.ChangeSourceUrlToGitHub(new Uri(inputPath)); // so updating existing clients doesn't break - var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml"; - input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); - logger.LogInformation("loaded description from remote source"); - } - catch (HttpRequestException ex) - { - throw new InvalidOperationException($"Could not download the file at {inputPath}, reason: {ex.Message}", ex); - } - else - try - { -#pragma warning disable CA2000 // disposed by caller - var inMemoryStream = new MemoryStream(); - using (await localFilesLock.LockAsync(inputPath, cancellationToken).ConfigureAwait(false)) - {// To avoid deadlocking on update with multiple clients for the same local description - using var fileStream = new FileStream(inputPath, FileMode.Open); - await fileStream.CopyToAsync(inMemoryStream, cancellationToken).ConfigureAwait(false); - } - inMemoryStream.Position = 0; - input = inMemoryStream; - logger.LogInformation("loaded description from local source"); -#pragma warning restore CA2000 - } - catch (Exception ex) when (ex is FileNotFoundException || - ex is PathTooLongException || - ex is DirectoryNotFoundException || - ex is IOException || - ex is UnauthorizedAccessException || - ex is SecurityException || - ex is NotSupportedException) - { - throw new InvalidOperationException($"Could not open the file at {inputPath}, reason: {ex.Message}", ex); - } - stopwatch.Stop(); - logger.LogTrace("{Timestamp}ms: Read OpenAPI file {File}", stopwatch.ElapsedMilliseconds, inputPath); + var (input, isCopy) = await DownloadHelper.LoadStream(inputPath, httpClient, logger, config, localFilesLock, workspaceManagementService, useKiotaConfig, cancellationToken).ConfigureAwait(false); + isDescriptionFromWorkspaceCopy = isCopy; return input; } diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index 5d7268e836..39867089fb 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -3,11 +3,13 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using AsyncKeyedLock; using Kiota.Builder.Configuration; using Kiota.Builder.Extensions; using Kiota.Builder.Lock; @@ -22,10 +24,13 @@ public class WorkspaceManagementService { private readonly bool UseKiotaConfig; private readonly ILogger Logger; - public WorkspaceManagementService(ILogger logger, bool useKiotaConfig = false, string workingDirectory = "") + private readonly HttpClient HttpClient; + public WorkspaceManagementService(ILogger logger, HttpClient httpClient, bool useKiotaConfig = false, string workingDirectory = "") { ArgumentNullException.ThrowIfNull(logger); + ArgumentNullException.ThrowIfNull(httpClient); Logger = logger; + HttpClient = httpClient; UseKiotaConfig = useKiotaConfig; if (string.IsNullOrEmpty(workingDirectory)) workingDirectory = Directory.GetCurrentDirectory(); @@ -183,9 +188,16 @@ public async Task> MigrateFromLockFileAsync(string clientNam lockDirectory = Path.Combine(WorkingDirectory, lockDirectory); if (Path.GetRelativePath(WorkingDirectory, lockDirectory).StartsWith("..", StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException("The lock directory must be a subdirectory of the working directory"); + + if (!await workspaceConfigurationStorageService.IsInitializedAsync(cancellationToken).ConfigureAwait(false)) + await workspaceConfigurationStorageService.InitializeAsync(cancellationToken).ConfigureAwait(false); + var (wsConfig, apiManifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); - wsConfig ??= new WorkspaceConfiguration(); - apiManifest ??= new ApiManifestDocument("application"); //TODO get the application name + if (wsConfig is null) + throw new InvalidOperationException("The workspace configuration is not initialized"); + if (apiManifest is null) + throw new InvalidOperationException("The API manifest is not initialized"); + var lockFiles = Directory.GetFiles(lockDirectory, LockManagementService.LockFileName, SearchOption.AllDirectories); if (lockFiles.Length == 0) throw new InvalidOperationException("No lock file found in the specified directory"); @@ -212,12 +224,18 @@ public async Task> MigrateFromLockFileAsync(string clientNam var inputConfigurationHash = await GetConfigurationHashAsync(generationClientConfig, "migrated").ConfigureAwait(false); // because it's a migration, we don't want to calculate the exact hash since the description might have changed since the initial generation that created the lock file apiManifest.ApiDependencies.Add(configuration.ClientClassName, configuration.ToApiDependency(inputConfigurationHash, new()));//TODO get the resolved operations? - //TODO copy the description file + var (stream, _) = await DownloadHelper.LoadStream(configuration.OpenAPIFilePath, HttpClient, Logger, configuration, localFilesLock, null, false, cancellationToken).ConfigureAwait(false); + await descriptionStorageService.UpdateDescriptionAsync(configuration.ClientClassName, stream, string.Empty, cancellationToken).ConfigureAwait(false); lockManagementService.DeleteLockFile(Path.GetDirectoryName(configuration.OpenAPIFilePath)!); } await workspaceConfigurationStorageService.UpdateWorkspaceConfigurationAsync(wsConfig, apiManifest, cancellationToken).ConfigureAwait(false); return clientsGenerationConfigurations.OfType().Select(static x => x.ClientClassName); } + private static readonly AsyncKeyedLocker localFilesLock = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); private async Task LoadConfigurationFromLockAsync(string clientName, string lockFilePath, CancellationToken cancellationToken) { if (Path.GetDirectoryName(lockFilePath) is not string lockFileDirectory) diff --git a/src/kiota/Handlers/Client/RemoveHandler.cs b/src/kiota/Handlers/Client/RemoveHandler.cs index 1ad2352cf7..693c50f03d 100644 --- a/src/kiota/Handlers/Client/RemoveHandler.cs +++ b/src/kiota/Handlers/Client/RemoveHandler.cs @@ -29,7 +29,7 @@ public override async Task InvokeAsync(InvocationContext context) try { await CheckForNewVersionAsync(logger, cancellationToken).ConfigureAwait(false); - var workspaceManagementService = new WorkspaceManagementService(logger, true); + var workspaceManagementService = new WorkspaceManagementService(logger, httpClient, true); await workspaceManagementService.RemoveClientAsync(className, cleanOutput, cancellationToken).ConfigureAwait(false); return 0; } diff --git a/src/kiota/Handlers/Config/MigrateHandler.cs b/src/kiota/Handlers/Config/MigrateHandler.cs index a1b05ade6f..1be224cfc8 100644 --- a/src/kiota/Handlers/Config/MigrateHandler.cs +++ b/src/kiota/Handlers/Config/MigrateHandler.cs @@ -33,7 +33,7 @@ public override async Task InvokeAsync(InvocationContext context) { try { - var workspaceManagementService = new WorkspaceManagementService(logger, true, workingDirectory); + var workspaceManagementService = new WorkspaceManagementService(logger, httpClient, true, workingDirectory); var clientNames = await workspaceManagementService.MigrateFromLockFileAsync(clientName, lockDirectory, cancellationToken).ConfigureAwait(false); if (!clientNames.Any()) { diff --git a/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs b/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs index b4fb935c1d..cfdb3e0ac2 100644 --- a/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs +++ b/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Net.Http; using System.Threading.Tasks; using Kiota.Builder.Configuration; using Kiota.Builder.WorkspaceManagement; @@ -12,19 +13,21 @@ namespace Kiota.Builder.Tests.WorkspaceManagement; public sealed class WorkspaceManagementServiceTests : IDisposable { private readonly string tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + private readonly HttpClient httpClient = new(); [Fact] public void Defensive() { - Assert.Throws(() => new WorkspaceManagementService(null)); + Assert.Throws(() => new WorkspaceManagementService(null, httpClient)); + Assert.Throws(() => new WorkspaceManagementService(Mock.Of(), null)); } [InlineData(true)] [InlineData(false)] [Theory] public async Task IsClientPresentReturnsFalseOnNoClient(bool usesConfig) { - var mockLogger = new Mock(); + var mockLogger = Mock.Of(); Directory.CreateDirectory(tempPath); - var service = new WorkspaceManagementService(mockLogger.Object, usesConfig, tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, usesConfig, tempPath); var result = await service.IsClientPresent("clientName"); Assert.False(result); } @@ -33,9 +36,9 @@ public async Task IsClientPresentReturnsFalseOnNoClient(bool usesConfig) [Theory] public async Task ShouldGenerateReturnsTrue(bool usesConfig) { - var mockLogger = new Mock(); + var mockLogger = Mock.Of(); Directory.CreateDirectory(tempPath); - var service = new WorkspaceManagementService(mockLogger.Object, usesConfig, tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, usesConfig, tempPath); var configuration = new GenerationConfiguration { ClientClassName = "clientName", @@ -50,9 +53,9 @@ public async Task ShouldGenerateReturnsTrue(bool usesConfig) [Theory] public async Task ShouldGenerateReturnsFalse(bool usesConfig) { - var mockLogger = new Mock(); + var mockLogger = Mock.Of(); Directory.CreateDirectory(tempPath); - var service = new WorkspaceManagementService(mockLogger.Object, usesConfig, tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, usesConfig, tempPath); var configuration = new GenerationConfiguration { ClientClassName = "clientName", @@ -68,9 +71,9 @@ public async Task ShouldGenerateReturnsFalse(bool usesConfig) [Fact] public async Task RemovesAClient() { - var mockLogger = new Mock(); + var mockLogger = Mock.Of(); Directory.CreateDirectory(tempPath); - var service = new WorkspaceManagementService(mockLogger.Object, true, tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, true, tempPath); var configuration = new GenerationConfiguration { ClientClassName = "clientName", @@ -89,5 +92,6 @@ public void Dispose() { if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + httpClient.Dispose(); } } From 5c8da95ab0d1a5bedad3cdd406cf5be512d74215 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 09:28:24 -0500 Subject: [PATCH 04/17] - updates specification regarding requests behaviour Signed-off-by: Vincent Biret --- specs/cli/config-migrate.md | 56 ++----------------- .../WorkspaceManagementService.cs | 2 +- 2 files changed, 5 insertions(+), 53 deletions(-) diff --git a/specs/cli/config-migrate.md b/specs/cli/config-migrate.md index 86b3fa2bb8..4223a65aec 100644 --- a/specs/cli/config-migrate.md +++ b/specs/cli/config-migrate.md @@ -2,6 +2,8 @@ This command is valuable in cases where a code base was created with Kiota v1.0 and needs to be migrated to the latest version of Kiota. The `kiota config migrate` command will identify and locate the closest `kiota-config.json` file available. If a file can't be found, it would initialize a new `kiota-config.json` file. Then, it would identify all `kiota-lock.json` files that are within this folder structure and add each of them to the `kiota-config.json` file. Adding the clients to the `kiota-config.json` file would not trigger the generation as it only affects the `kiota-config.json` file. The `kiota client generate` command would need to be executed to generate the code for the clients. +The API manifest won't contain any request after the migration since it could lead to misalignments between the generated client and the reported requests if the description has changed between the initial generation of the client and the migration. To get the requests populated, the user will need to use the generate command. + In the case where conflicting API client names would be migrated, the command will error out and invite the user to re-run the command providing more context for the `--client-name` parameter. ## Parameters @@ -80,64 +82,14 @@ _The resulting `apimanifest.json` file will look like this:_ "apiDescriptionUrl": "https://aka.ms/graph/v1.0/openapi.yaml", "apiDeploymentBaseUrl": "https://graph.microsoft.com", "apiDescriptionVersion": "v1.0", - "requests": [ - { - "method": "GET", - "uriTemplate": "/users" - }, - { - "method": "POST", - "uriTemplate": "/users" - }, - { - "method": "GET", - "uriTemplate": "/users/$count" - }, - { - "method": "GET", - "uriTemplate": "/users/{user-id}" - }, - { - "method": "PATCH", - "uriTemplate": "/users/{user-id}" - }, - { - "method": "DELETE", - "uriTemplate": "/users/{user-id}" - } - ] + "requests": [] }, "pythonGraphServiceClient": { "x-ms-kiotaHash": "9EDF8506CB74FE44...", "apiDescriptionUrl": "https://aka.ms/graph/v1.0/openapi.yaml", "apiDeploymentBaseUrl": "https://graph.microsoft.com", "apiDescriptionVersion": "v1.0", - "requests": [ - { - "method": "GET", - "uriTemplate": "/users" - }, - { - "method": "POST", - "uriTemplate": "/users" - }, - { - "method": "GET", - "uriTemplate": "/users/$count" - }, - { - "method": "GET", - "uriTemplate": "/users/{user-id}" - }, - { - "method": "PATCH", - "uriTemplate": "/users/{user-id}" - }, - { - "method": "DELETE", - "uriTemplate": "/users/{user-id}" - } - ] + "requests": [] } } } diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index 39867089fb..a4bbfc8be0 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -221,7 +221,7 @@ public async Task> MigrateFromLockFileAsync(string clientNam continue; } wsConfig.Clients.Add(configuration.ClientClassName, generationClientConfig); - var inputConfigurationHash = await GetConfigurationHashAsync(generationClientConfig, "migrated").ConfigureAwait(false); + var inputConfigurationHash = await GetConfigurationHashAsync(generationClientConfig, "migrated-pending-generate").ConfigureAwait(false); // because it's a migration, we don't want to calculate the exact hash since the description might have changed since the initial generation that created the lock file apiManifest.ApiDependencies.Add(configuration.ClientClassName, configuration.ToApiDependency(inputConfigurationHash, new()));//TODO get the resolved operations? var (stream, _) = await DownloadHelper.LoadStream(configuration.OpenAPIFilePath, HttpClient, Logger, configuration, localFilesLock, null, false, cancellationToken).ConfigureAwait(false); From da0ccf5ea710c4553986a17fb9c6d845b1444d01 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 09:36:48 -0500 Subject: [PATCH 05/17] makes the client name optional for the migrate command Signed-off-by: Vincent Biret --- src/kiota/KiotaClientCommands.cs | 4 ++-- src/kiota/KiotaConfigCommands.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kiota/KiotaClientCommands.cs b/src/kiota/KiotaClientCommands.cs index 97dba4e76b..07a3f998e4 100644 --- a/src/kiota/KiotaClientCommands.cs +++ b/src/kiota/KiotaClientCommands.cs @@ -20,11 +20,11 @@ private static Option GetSkipGenerationOption() skipGeneration.AddAlias("--sg"); return skipGeneration; } - internal static Option GetClientNameOption() + internal static Option GetClientNameOption(bool required = true) { var clientName = new Option("--client-name", "The name of the client to manage") { - IsRequired = true, + IsRequired = required, }; clientName.AddAlias("--cn"); return clientName; diff --git a/src/kiota/KiotaConfigCommands.cs b/src/kiota/KiotaConfigCommands.cs index abb272ea8d..59e0e8f63d 100644 --- a/src/kiota/KiotaConfigCommands.cs +++ b/src/kiota/KiotaConfigCommands.cs @@ -28,7 +28,7 @@ private static Command GetMigrateCommand() { var logLevelOption = KiotaHost.GetLogLevelOption(); var lockDirectoryOption = GetLockDirectoryOption(); - var classOption = KiotaClientCommands.GetClientNameOption(); + var classOption = KiotaClientCommands.GetClientNameOption(false); var command = new Command("migrate", "Migrates a kiota lock file to a Kiota configuration") { logLevelOption, From 28a7a3e8a12f28e2c9769b44da54fad2b0731557 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 09:52:58 -0500 Subject: [PATCH 06/17] - defaults the manifest on migration Signed-off-by: Vincent Biret --- .../WorkspaceManagement/WorkspaceManagementService.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index a4bbfc8be0..b604eb5517 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -54,7 +54,7 @@ public async Task UpdateStateFromConfigurationAsync(GenerationConfiguration gene { var (wsConfig, manifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); wsConfig ??= new WorkspaceConfiguration(); - manifest ??= new ApiManifestDocument("application"); //TODO get the application name + manifest ??= defaultManifest; var generationClientConfig = new ApiClientConfiguration(generationConfiguration); generationClientConfig.NormalizePaths(WorkingDirectory); wsConfig.Clients.AddOrReplace(generationConfiguration.ClientClassName, generationClientConfig); @@ -73,6 +73,7 @@ public async Task UpdateStateFromConfigurationAsync(GenerationConfiguration gene await lockManagementService.WriteLockFileAsync(generationConfiguration.OutputPath, configurationLock, cancellationToken).ConfigureAwait(false); } } + private static ApiManifestDocument defaultManifest => new("application"); //TODO get the application name public async Task RestoreStateAsync(string outputPath, CancellationToken cancellationToken = default) { if (UseKiotaConfig) @@ -195,8 +196,7 @@ public async Task> MigrateFromLockFileAsync(string clientNam var (wsConfig, apiManifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); if (wsConfig is null) throw new InvalidOperationException("The workspace configuration is not initialized"); - if (apiManifest is null) - throw new InvalidOperationException("The API manifest is not initialized"); + apiManifest ??= defaultManifest; var lockFiles = Directory.GetFiles(lockDirectory, LockManagementService.LockFileName, SearchOption.AllDirectories); if (lockFiles.Length == 0) From 0d55f1e59194c4adbc908f0453333a5e7d733b37 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 10:08:24 -0500 Subject: [PATCH 07/17] - code linting Signed-off-by: Vincent Biret --- .../WorkspaceManagementService.cs | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index b604eb5517..35a62ae772 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -52,9 +52,7 @@ public async Task UpdateStateFromConfigurationAsync(GenerationConfiguration gene ArgumentNullException.ThrowIfNull(generationConfiguration); if (UseKiotaConfig) { - var (wsConfig, manifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); - wsConfig ??= new WorkspaceConfiguration(); - manifest ??= defaultManifest; + var (wsConfig, manifest) = await LoadConfigurationAndManifestAsync(cancellationToken).ConfigureAwait(false); var generationClientConfig = new ApiClientConfiguration(generationConfiguration); generationClientConfig.NormalizePaths(WorkingDirectory); wsConfig.Clients.AddOrReplace(generationConfiguration.ClientClassName, generationClientConfig); @@ -73,7 +71,6 @@ public async Task UpdateStateFromConfigurationAsync(GenerationConfiguration gene await lockManagementService.WriteLockFileAsync(generationConfiguration.OutputPath, configurationLock, cancellationToken).ConfigureAwait(false); } } - private static ApiManifestDocument defaultManifest => new("application"); //TODO get the application name public async Task RestoreStateAsync(string outputPath, CancellationToken cancellationToken = default) { if (UseKiotaConfig) @@ -179,8 +176,18 @@ private static string ConvertByteArrayToString(byte[] hash) return sb.ToString(); } + private async Task<(WorkspaceConfiguration, ApiManifestDocument)> LoadConfigurationAndManifestAsync(CancellationToken cancellationToken) + { + if (!await workspaceConfigurationStorageService.IsInitializedAsync(cancellationToken).ConfigureAwait(false)) + await workspaceConfigurationStorageService.InitializeAsync(cancellationToken).ConfigureAwait(false); - public async Task> MigrateFromLockFileAsync(string clientName, string lockDirectory, CancellationToken cancellationToken = default) + var (wsConfig, apiManifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); + if (wsConfig is null) + throw new InvalidOperationException("The workspace configuration is not initialized"); + apiManifest ??= new("application"); //TODO get the application name + return (wsConfig, apiManifest); + } + private async Task> LoadGenerationConfigurationsFromLockFilesAsync(string lockDirectory, string clientName, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrEmpty(lockDirectory); if (!UseKiotaConfig) @@ -190,14 +197,6 @@ public async Task> MigrateFromLockFileAsync(string clientNam if (Path.GetRelativePath(WorkingDirectory, lockDirectory).StartsWith("..", StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException("The lock directory must be a subdirectory of the working directory"); - if (!await workspaceConfigurationStorageService.IsInitializedAsync(cancellationToken).ConfigureAwait(false)) - await workspaceConfigurationStorageService.InitializeAsync(cancellationToken).ConfigureAwait(false); - - var (wsConfig, apiManifest) = await workspaceConfigurationStorageService.GetWorkspaceConfigurationAsync(cancellationToken).ConfigureAwait(false); - if (wsConfig is null) - throw new InvalidOperationException("The workspace configuration is not initialized"); - apiManifest ??= defaultManifest; - var lockFiles = Directory.GetFiles(lockDirectory, LockManagementService.LockFileName, SearchOption.AllDirectories); if (lockFiles.Length == 0) throw new InvalidOperationException("No lock file found in the specified directory"); @@ -209,8 +208,14 @@ public async Task> MigrateFromLockFileAsync(string clientNam clientsGenerationConfigurations.Add(await LoadConfigurationFromLockAsync(clientNamePassed ? clientName : string.Empty, lockFiles[0], cancellationToken).ConfigureAwait(false)); else clientsGenerationConfigurations.AddRange(await Task.WhenAll(lockFiles.Select(x => LoadConfigurationFromLockAsync(string.Empty, x, cancellationToken))).ConfigureAwait(false)); - var loadedConfigurations = clientsGenerationConfigurations.OfType().ToArray(); - foreach (var configuration in loadedConfigurations) + return clientsGenerationConfigurations.OfType().ToList(); + } + public async Task> MigrateFromLockFileAsync(string clientName, string lockDirectory, CancellationToken cancellationToken = default) + { + var (wsConfig, apiManifest) = await LoadConfigurationAndManifestAsync(cancellationToken).ConfigureAwait(false); + + var clientsGenerationConfigurations = await LoadGenerationConfigurationsFromLockFilesAsync(lockDirectory, clientName, cancellationToken).ConfigureAwait(false); + foreach (var configuration in clientsGenerationConfigurations.ToArray()) //to avoid modifying the collection as we iterate and remove some entries { var generationClientConfig = new ApiClientConfiguration(configuration); generationClientConfig.NormalizePaths(WorkingDirectory); From f27b8dc0d17c59765527fa202907f7b1d0b7f900 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 10:11:29 -0500 Subject: [PATCH 08/17] - code linting Signed-off-by: Vincent Biret --- src/Kiota.Builder/DownloadHelper.cs | 7 ++++++- src/Kiota.Builder/KiotaBuilder.cs | 8 +------- .../WorkspaceManagement/WorkspaceManagementService.cs | 7 +------ 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/Kiota.Builder/DownloadHelper.cs b/src/Kiota.Builder/DownloadHelper.cs index d2834006ab..6f9dfa3432 100644 --- a/src/Kiota.Builder/DownloadHelper.cs +++ b/src/Kiota.Builder/DownloadHelper.cs @@ -16,7 +16,12 @@ namespace Kiota.Builder; internal static class DownloadHelper { - internal static async Task<(Stream, bool)> LoadStream(string inputPath, HttpClient httpClient, ILogger logger, GenerationConfiguration config, AsyncKeyedLocker localFilesLock, WorkspaceManagementService? workspaceManagementService = default, bool useKiotaConfig = false, CancellationToken cancellationToken = default) + private static readonly AsyncKeyedLocker localFilesLock = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); + internal static async Task<(Stream, bool)> LoadStream(string inputPath, HttpClient httpClient, ILogger logger, GenerationConfiguration config, WorkspaceManagementService? workspaceManagementService = default, bool useKiotaConfig = false, CancellationToken cancellationToken = default) { var stopwatch = new Stopwatch(); stopwatch.Start(); diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index 9cd0706c7a..30d4a7a580 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -391,16 +391,10 @@ private void StopLogAndReset(Stopwatch sw, string prefix) logger.LogDebug("{Prefix} {SwElapsed}", prefix, sw.Elapsed); sw.Reset(); } - - private static readonly AsyncKeyedLocker localFilesLock = new(o => - { - o.PoolSize = 20; - o.PoolInitialFill = 1; - }); private bool isDescriptionFromWorkspaceCopy; private async Task LoadStream(string inputPath, CancellationToken cancellationToken) { - var (input, isCopy) = await DownloadHelper.LoadStream(inputPath, httpClient, logger, config, localFilesLock, workspaceManagementService, useKiotaConfig, cancellationToken).ConfigureAwait(false); + var (input, isCopy) = await DownloadHelper.LoadStream(inputPath, httpClient, logger, config, workspaceManagementService, useKiotaConfig, cancellationToken).ConfigureAwait(false); isDescriptionFromWorkspaceCopy = isCopy; return input; } diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index 35a62ae772..d45182bb38 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -229,18 +229,13 @@ public async Task> MigrateFromLockFileAsync(string clientNam var inputConfigurationHash = await GetConfigurationHashAsync(generationClientConfig, "migrated-pending-generate").ConfigureAwait(false); // because it's a migration, we don't want to calculate the exact hash since the description might have changed since the initial generation that created the lock file apiManifest.ApiDependencies.Add(configuration.ClientClassName, configuration.ToApiDependency(inputConfigurationHash, new()));//TODO get the resolved operations? - var (stream, _) = await DownloadHelper.LoadStream(configuration.OpenAPIFilePath, HttpClient, Logger, configuration, localFilesLock, null, false, cancellationToken).ConfigureAwait(false); + var (stream, _) = await DownloadHelper.LoadStream(configuration.OpenAPIFilePath, HttpClient, Logger, configuration, null, false, cancellationToken).ConfigureAwait(false); await descriptionStorageService.UpdateDescriptionAsync(configuration.ClientClassName, stream, string.Empty, cancellationToken).ConfigureAwait(false); lockManagementService.DeleteLockFile(Path.GetDirectoryName(configuration.OpenAPIFilePath)!); } await workspaceConfigurationStorageService.UpdateWorkspaceConfigurationAsync(wsConfig, apiManifest, cancellationToken).ConfigureAwait(false); return clientsGenerationConfigurations.OfType().Select(static x => x.ClientClassName); } - private static readonly AsyncKeyedLocker localFilesLock = new(o => - { - o.PoolSize = 20; - o.PoolInitialFill = 1; - }); private async Task LoadConfigurationFromLockAsync(string clientName, string lockFilePath, CancellationToken cancellationToken) { if (Path.GetDirectoryName(lockFilePath) is not string lockFileDirectory) From e3c0ab6d32c4ff6cf2378073ea1541a27618a052 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 10:23:43 -0500 Subject: [PATCH 09/17] - moves api url logic outside of builder Signed-off-by: Vincent Biret --- .../Extensions/OpenApiDocumentExtensions.cs | 21 ++++++++++++ src/Kiota.Builder/KiotaBuilder.cs | 34 ++++--------------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs b/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs index e0abc3cb94..dcfdb31c5b 100644 --- a/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs +++ b/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using System.Linq; using System.Threading.Tasks; +using Kiota.Builder.EqualityComparers; using Microsoft.OpenApi.Models; namespace Kiota.Builder.Extensions; @@ -26,4 +27,24 @@ internal static void InitializeInheritanceIndex(this OpenApiDocument openApiDocu }); } } + internal static string? GetAPIRootUrl(this OpenApiDocument openApiDocument, string openAPIFilePath) + { + if (openApiDocument == null) return null; + var candidateUrl = openApiDocument.Servers + .GroupBy(static x => x, new OpenApiServerComparer()) //group by protocol relative urls + .FirstOrDefault() + ?.OrderByDescending(static x => x?.Url, StringComparer.OrdinalIgnoreCase) // prefer https over http + ?.FirstOrDefault() + ?.Url; + if (string.IsNullOrEmpty(candidateUrl)) + return null; + else if (!candidateUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase) && + openAPIFilePath.StartsWith("http", StringComparison.OrdinalIgnoreCase) && + Uri.TryCreate(openAPIFilePath, new(), out var filePathUri) && + Uri.TryCreate(filePathUri, candidateUrl, out var candidateUri)) + { + candidateUrl = candidateUri.ToString(); + } + return candidateUrl.TrimEnd(KiotaBuilder.ForwardSlash); + } } diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index 30d4a7a580..2d703d1933 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -357,33 +357,11 @@ internal void FilterPathsByPatterns(OpenApiDocument doc) } internal void SetApiRootUrl() { - if (openApiDocument == null) return; - var candidateUrl = openApiDocument.Servers - .GroupBy(static x => x, new OpenApiServerComparer()) //group by protocol relative urls - .FirstOrDefault() - ?.OrderByDescending(static x => x?.Url, StringComparer.OrdinalIgnoreCase) // prefer https over http - ?.FirstOrDefault() - ?.Url; - if (string.IsNullOrEmpty(candidateUrl)) - { + var candidateUrl = openApiDocument?.GetAPIRootUrl(config.OpenAPIFilePath); + if (candidateUrl is null) logger.LogWarning("No server url found in the OpenAPI document. The base url will need to be set when using the client."); - return; - } - else if (!candidateUrl.StartsWith("http", StringComparison.OrdinalIgnoreCase) && config.OpenAPIFilePath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) - { - try - { - candidateUrl = new Uri(new Uri(config.OpenAPIFilePath), candidateUrl).ToString(); - } -#pragma warning disable CA1031 - catch (Exception ex) -#pragma warning restore CA1031 - { - logger.LogWarning(ex, "Could not resolve the server url from the OpenAPI document. The base url will need to be set when using the client."); - return; - } - } - config.ApiRootUrl = candidateUrl.TrimEnd(ForwardSlash); + else + config.ApiRootUrl = candidateUrl; } private void StopLogAndReset(Stopwatch sw, string prefix) { @@ -399,8 +377,8 @@ private async Task LoadStream(string inputPath, CancellationToken cancel return input; } - private const char ForwardSlash = '/'; - public async Task CreateOpenApiDocumentAsync(Stream input, bool generating = false, CancellationToken cancellationToken = default) + internal const char ForwardSlash = '/'; + internal async Task CreateOpenApiDocumentAsync(Stream input, bool generating = false, CancellationToken cancellationToken = default) { var stopwatch = new Stopwatch(); stopwatch.Start(); From 9e73e70ba2c378251b4518c0e277cf38912fe6f3 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 11:10:17 -0500 Subject: [PATCH 10/17] - sets the base url for migration Signed-off-by: Vincent Biret --- src/Kiota.Builder/DownloadHelper.cs | 85 ---------- src/Kiota.Builder/KiotaBuilder.cs | 61 +------ .../OpenApiDocumentDownloadService.cs | 155 ++++++++++++++++++ .../WorkspaceManagementService.cs | 39 +++-- 4 files changed, 187 insertions(+), 153 deletions(-) delete mode 100644 src/Kiota.Builder/DownloadHelper.cs create mode 100644 src/Kiota.Builder/OpenApiDocumentDownloadService.cs diff --git a/src/Kiota.Builder/DownloadHelper.cs b/src/Kiota.Builder/DownloadHelper.cs deleted file mode 100644 index 6f9dfa3432..0000000000 --- a/src/Kiota.Builder/DownloadHelper.cs +++ /dev/null @@ -1,85 +0,0 @@ -using System; -using System.Diagnostics; -using System.IO; -using System.Net.Http; -using System.Security; -using System.Threading; -using System.Threading.Tasks; -using AsyncKeyedLock; -using Kiota.Builder.Caching; -using Kiota.Builder.Configuration; -using Kiota.Builder.Extensions; -using Kiota.Builder.SearchProviders.APIsGuru; -using Kiota.Builder.WorkspaceManagement; -using Microsoft.Extensions.Logging; - -namespace Kiota.Builder; -internal static class DownloadHelper -{ - private static readonly AsyncKeyedLocker localFilesLock = new(o => - { - o.PoolSize = 20; - o.PoolInitialFill = 1; - }); - internal static async Task<(Stream, bool)> LoadStream(string inputPath, HttpClient httpClient, ILogger logger, GenerationConfiguration config, WorkspaceManagementService? workspaceManagementService = default, bool useKiotaConfig = false, CancellationToken cancellationToken = default) - { - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - inputPath = inputPath.Trim(); - - Stream input; - var isDescriptionFromWorkspaceCopy = false; - if (useKiotaConfig && - config.Operation is ClientOperation.Edit or ClientOperation.Add && - workspaceManagementService is not null && - await workspaceManagementService.GetDescriptionCopyAsync(config.ClientClassName, inputPath, cancellationToken).ConfigureAwait(false) is { } descriptionStream) - { - logger.LogInformation("loaded description from the workspace copy"); - input = descriptionStream; - isDescriptionFromWorkspaceCopy = true; - } - else if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) - try - { - var cachingProvider = new DocumentCachingProvider(httpClient, logger) - { - ClearCache = config.ClearCache, - }; - var targetUri = APIsGuruSearchProvider.ChangeSourceUrlToGitHub(new Uri(inputPath)); // so updating existing clients doesn't break - var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml"; - input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); - logger.LogInformation("loaded description from remote source"); - } - catch (HttpRequestException ex) - { - throw new InvalidOperationException($"Could not download the file at {inputPath}, reason: {ex.Message}", ex); - } - else - try - { - var inMemoryStream = new MemoryStream(); - using (await localFilesLock.LockAsync(inputPath, cancellationToken).ConfigureAwait(false)) - {// To avoid deadlocking on update with multiple clients for the same local description - using var fileStream = new FileStream(inputPath, FileMode.Open); - await fileStream.CopyToAsync(inMemoryStream, cancellationToken).ConfigureAwait(false); - } - inMemoryStream.Position = 0; - input = inMemoryStream; - logger.LogInformation("loaded description from local source"); - } - catch (Exception ex) when (ex is FileNotFoundException || - ex is PathTooLongException || - ex is DirectoryNotFoundException || - ex is IOException || - ex is UnauthorizedAccessException || - ex is SecurityException || - ex is NotSupportedException) - { - throw new InvalidOperationException($"Could not open the file at {inputPath}, reason: {ex.Message}", ex); - } - stopwatch.Stop(); - logger.LogTrace("{Timestamp}ms: Read OpenAPI file {File}", stopwatch.ElapsedMilliseconds, inputPath); - return (input, isDescriptionFromWorkspaceCopy); - } -} diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index 2d703d1933..d150c807e0 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -11,7 +11,6 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using AsyncKeyedLock; using DotNet.Globbing; using Kiota.Builder.Caching; using Kiota.Builder.CodeDOM; @@ -24,7 +23,6 @@ using Kiota.Builder.Manifest; using Kiota.Builder.OpenApiExtensions; using Kiota.Builder.Refiners; -using Kiota.Builder.Validation; using Kiota.Builder.WorkspaceManagement; using Kiota.Builder.Writers; using Microsoft.Extensions.Logging; @@ -32,9 +30,7 @@ using Microsoft.OpenApi.ApiManifest; using Microsoft.OpenApi.MicrosoftExtensions; using Microsoft.OpenApi.Models; -using Microsoft.OpenApi.Readers; using Microsoft.OpenApi.Services; -using Microsoft.OpenApi.Validations; using HttpMethod = Kiota.Builder.CodeDOM.HttpMethod; [assembly: InternalsVisibleTo("Kiota.Builder.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100957cb48387b2a5f54f5ce39255f18f26d32a39990db27cf48737afc6bc62759ba996b8a2bfb675d4e39f3d06ecb55a178b1b4031dcb2a767e29977d88cce864a0d16bfc1b3bebb0edf9fe285f10fffc0a85f93d664fa05af07faa3aad2e545182dbf787e3fd32b56aca95df1a3c4e75dec164a3f1a4c653d971b01ffc39eb3c4")] @@ -64,7 +60,9 @@ public KiotaBuilder(ILogger logger, GenerationConfiguration config var workingDirectory = Directory.GetCurrentDirectory(); workspaceManagementService = new WorkspaceManagementService(logger, client, useKiotaConfig, workingDirectory); this.useKiotaConfig = useKiotaConfig; + openApiDocumentDownloadService = new OpenApiDocumentDownloadService(client, logger); } + private readonly OpenApiDocumentDownloadService openApiDocumentDownloadService; private readonly bool useKiotaConfig; private async Task CleanOutputDirectory(CancellationToken cancellationToken) { @@ -372,64 +370,15 @@ private void StopLogAndReset(Stopwatch sw, string prefix) private bool isDescriptionFromWorkspaceCopy; private async Task LoadStream(string inputPath, CancellationToken cancellationToken) { - var (input, isCopy) = await DownloadHelper.LoadStream(inputPath, httpClient, logger, config, workspaceManagementService, useKiotaConfig, cancellationToken).ConfigureAwait(false); + var (input, isCopy) = await openApiDocumentDownloadService.LoadStreamAsync(inputPath, config, workspaceManagementService, useKiotaConfig, cancellationToken).ConfigureAwait(false); isDescriptionFromWorkspaceCopy = isCopy; return input; } internal const char ForwardSlash = '/'; - internal async Task CreateOpenApiDocumentAsync(Stream input, bool generating = false, CancellationToken cancellationToken = default) + internal Task CreateOpenApiDocumentAsync(Stream input, bool generating = false, CancellationToken cancellationToken = default) { - var stopwatch = new Stopwatch(); - stopwatch.Start(); - logger.LogTrace("Parsing OpenAPI file"); - var ruleSet = config.DisabledValidationRules.Contains(ValidationRuleSetExtensions.AllValidationRule) ? - ValidationRuleSet.GetEmptyRuleSet() : - ValidationRuleSet.GetDefaultRuleSet(); //workaround since validation rule set doesn't support clearing rules - if (generating) - ruleSet.AddKiotaValidationRules(config); - var settings = new OpenApiReaderSettings - { - RuleSet = ruleSet, - }; - settings.AddMicrosoftExtensionParsers(); - settings.ExtensionParsers.TryAdd(OpenApiKiotaExtension.Name, static (i, _) => OpenApiKiotaExtension.Parse(i)); - try - { - var rawUri = config.OpenAPIFilePath.TrimEnd(ForwardSlash); - var lastSlashIndex = rawUri.LastIndexOf(ForwardSlash); - if (lastSlashIndex < 0) - lastSlashIndex = rawUri.Length - 1; - var documentUri = new Uri(rawUri[..lastSlashIndex]); - settings.BaseUrl = documentUri; - settings.LoadExternalRefs = true; - } -#pragma warning disable CA1031 - catch -#pragma warning restore CA1031 - { - // couldn't parse the URL, it's probably a local file - } - var reader = new OpenApiStreamReader(settings); - var readResult = await reader.ReadAsync(input, cancellationToken).ConfigureAwait(false); - stopwatch.Stop(); - if (generating) - foreach (var warning in readResult.OpenApiDiagnostic.Warnings) - logger.LogWarning("OpenAPI warning: {Pointer} - {Warning}", warning.Pointer, warning.Message); - if (readResult.OpenApiDiagnostic.Errors.Any()) - { - logger.LogTrace("{Timestamp}ms: Parsed OpenAPI with errors. {Count} paths found.", stopwatch.ElapsedMilliseconds, readResult.OpenApiDocument?.Paths?.Count ?? 0); - foreach (var parsingError in readResult.OpenApiDiagnostic.Errors) - { - logger.LogError("OpenAPI error: {Pointer} - {Message}", parsingError.Pointer, parsingError.Message); - } - } - else - { - logger.LogTrace("{Timestamp}ms: Parsed OpenAPI successfully. {Count} paths found.", stopwatch.ElapsedMilliseconds, readResult.OpenApiDocument?.Paths?.Count ?? 0); - } - - return readResult.OpenApiDocument; + return openApiDocumentDownloadService.GetDocumentFromStreamAsync(input, config, generating, cancellationToken); } public static string GetDeeperMostCommonNamespaceNameForModels(OpenApiDocument document) { diff --git a/src/Kiota.Builder/OpenApiDocumentDownloadService.cs b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs new file mode 100644 index 0000000000..2ff8c3d180 --- /dev/null +++ b/src/Kiota.Builder/OpenApiDocumentDownloadService.cs @@ -0,0 +1,155 @@ +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Security; +using System.Threading; +using System.Threading.Tasks; +using AsyncKeyedLock; +using Kiota.Builder.Caching; +using Kiota.Builder.Configuration; +using Kiota.Builder.Extensions; +using Kiota.Builder.OpenApiExtensions; +using Kiota.Builder.SearchProviders.APIsGuru; +using Kiota.Builder.Validation; +using Kiota.Builder.WorkspaceManagement; +using Microsoft.Extensions.Logging; +using Microsoft.OpenApi.Models; +using Microsoft.OpenApi.Readers; +using Microsoft.OpenApi.Validations; + +namespace Kiota.Builder; +internal class OpenApiDocumentDownloadService +{ + private readonly ILogger Logger; + private readonly HttpClient HttpClient; + public OpenApiDocumentDownloadService(HttpClient httpClient, ILogger logger) + { + ArgumentNullException.ThrowIfNull(httpClient); + ArgumentNullException.ThrowIfNull(logger); + HttpClient = httpClient; + Logger = logger; + } + private static readonly AsyncKeyedLocker localFilesLock = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); + internal async Task<(Stream, bool)> LoadStreamAsync(string inputPath, GenerationConfiguration config, WorkspaceManagementService? workspaceManagementService = default, bool useKiotaConfig = false, CancellationToken cancellationToken = default) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + inputPath = inputPath.Trim(); + + Stream input; + var isDescriptionFromWorkspaceCopy = false; + if (useKiotaConfig && + config.Operation is ClientOperation.Edit or ClientOperation.Add && + workspaceManagementService is not null && + await workspaceManagementService.GetDescriptionCopyAsync(config.ClientClassName, inputPath, cancellationToken).ConfigureAwait(false) is { } descriptionStream) + { + Logger.LogInformation("loaded description from the workspace copy"); + input = descriptionStream; + isDescriptionFromWorkspaceCopy = true; + } + else if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + try + { + var cachingProvider = new DocumentCachingProvider(HttpClient, Logger) + { + ClearCache = config.ClearCache, + }; + var targetUri = APIsGuruSearchProvider.ChangeSourceUrlToGitHub(new Uri(inputPath)); // so updating existing clients doesn't break + var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml"; + input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); + Logger.LogInformation("loaded description from remote source"); + } + catch (HttpRequestException ex) + { + throw new InvalidOperationException($"Could not download the file at {inputPath}, reason: {ex.Message}", ex); + } + else + try + { + var inMemoryStream = new MemoryStream(); + using (await localFilesLock.LockAsync(inputPath, cancellationToken).ConfigureAwait(false)) + {// To avoid deadlocking on update with multiple clients for the same local description + using var fileStream = new FileStream(inputPath, FileMode.Open); + await fileStream.CopyToAsync(inMemoryStream, cancellationToken).ConfigureAwait(false); + } + inMemoryStream.Position = 0; + input = inMemoryStream; + Logger.LogInformation("loaded description from local source"); + } + catch (Exception ex) when (ex is FileNotFoundException || + ex is PathTooLongException || + ex is DirectoryNotFoundException || + ex is IOException || + ex is UnauthorizedAccessException || + ex is SecurityException || + ex is NotSupportedException) + { + throw new InvalidOperationException($"Could not open the file at {inputPath}, reason: {ex.Message}", ex); + } + stopwatch.Stop(); + Logger.LogTrace("{Timestamp}ms: Read OpenAPI file {File}", stopwatch.ElapsedMilliseconds, inputPath); + return (input, isDescriptionFromWorkspaceCopy); + } + + internal async Task GetDocumentFromStreamAsync(Stream input, GenerationConfiguration config, bool generating = false, CancellationToken cancellationToken = default) + { + var stopwatch = new Stopwatch(); + stopwatch.Start(); + Logger.LogTrace("Parsing OpenAPI file"); + var ruleSet = config.DisabledValidationRules.Contains(ValidationRuleSetExtensions.AllValidationRule) ? + ValidationRuleSet.GetEmptyRuleSet() : + ValidationRuleSet.GetDefaultRuleSet(); //workaround since validation rule set doesn't support clearing rules + if (generating) + ruleSet.AddKiotaValidationRules(config); + var settings = new OpenApiReaderSettings + { + RuleSet = ruleSet, + }; + settings.AddMicrosoftExtensionParsers(); + settings.ExtensionParsers.TryAdd(OpenApiKiotaExtension.Name, static (i, _) => OpenApiKiotaExtension.Parse(i)); + try + { + var rawUri = config.OpenAPIFilePath.TrimEnd(KiotaBuilder.ForwardSlash); + var lastSlashIndex = rawUri.LastIndexOf(KiotaBuilder.ForwardSlash); + if (lastSlashIndex < 0) + lastSlashIndex = rawUri.Length - 1; + var documentUri = new Uri(rawUri[..lastSlashIndex]); + settings.BaseUrl = documentUri; + settings.LoadExternalRefs = true; + settings.LeaveStreamOpen = true; + } +#pragma warning disable CA1031 + catch +#pragma warning restore CA1031 + { + // couldn't parse the URL, it's probably a local file + } + var reader = new OpenApiStreamReader(settings); + var readResult = await reader.ReadAsync(input, cancellationToken).ConfigureAwait(false); + stopwatch.Stop(); + if (generating) + foreach (var warning in readResult.OpenApiDiagnostic.Warnings) + Logger.LogWarning("OpenAPI warning: {Pointer} - {Warning}", warning.Pointer, warning.Message); + if (readResult.OpenApiDiagnostic.Errors.Any()) + { + Logger.LogTrace("{Timestamp}ms: Parsed OpenAPI with errors. {Count} paths found.", stopwatch.ElapsedMilliseconds, readResult.OpenApiDocument?.Paths?.Count ?? 0); + foreach (var parsingError in readResult.OpenApiDiagnostic.Errors) + { + Logger.LogError("OpenAPI error: {Pointer} - {Message}", parsingError.Pointer, parsingError.Message); + } + } + else + { + Logger.LogTrace("{Timestamp}ms: Parsed OpenAPI successfully. {Count} paths found.", stopwatch.ElapsedMilliseconds, readResult.OpenApiDocument?.Paths?.Count ?? 0); + } + + return readResult.OpenApiDocument; + } +} diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index d45182bb38..efca606b81 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -37,7 +37,9 @@ public WorkspaceManagementService(ILogger logger, HttpClient httpClient, bool us WorkingDirectory = workingDirectory; workspaceConfigurationStorageService = new(workingDirectory); descriptionStorageService = new(workingDirectory); + openApiDocumentDownloadService = new(HttpClient, Logger); } + private readonly OpenApiDocumentDownloadService openApiDocumentDownloadService; private readonly LockManagementService lockManagementService = new(); private readonly WorkspaceConfigurationStorageService workspaceConfigurationStorageService; private readonly DescriptionStorageService descriptionStorageService; @@ -215,23 +217,36 @@ public async Task> MigrateFromLockFileAsync(string clientNam var (wsConfig, apiManifest) = await LoadConfigurationAndManifestAsync(cancellationToken).ConfigureAwait(false); var clientsGenerationConfigurations = await LoadGenerationConfigurationsFromLockFilesAsync(lockDirectory, clientName, cancellationToken).ConfigureAwait(false); - foreach (var configuration in clientsGenerationConfigurations.ToArray()) //to avoid modifying the collection as we iterate and remove some entries + foreach (var generationConfiguration in clientsGenerationConfigurations.ToArray()) //to avoid modifying the collection as we iterate and remove some entries { - var generationClientConfig = new ApiClientConfiguration(configuration); - generationClientConfig.NormalizePaths(WorkingDirectory); - if (wsConfig.Clients.ContainsKey(configuration.ClientClassName)) + + if (wsConfig.Clients.ContainsKey(generationConfiguration.ClientClassName)) { - Logger.LogError("The client {ClientName} is already present in the configuration", configuration.ClientClassName); - clientsGenerationConfigurations.Remove(configuration); + Logger.LogError("The client {ClientName} is already present in the configuration", generationConfiguration.ClientClassName); + clientsGenerationConfigurations.Remove(generationConfiguration); continue; } - wsConfig.Clients.Add(configuration.ClientClassName, generationClientConfig); - var inputConfigurationHash = await GetConfigurationHashAsync(generationClientConfig, "migrated-pending-generate").ConfigureAwait(false); + var (stream, _) = await openApiDocumentDownloadService.LoadStreamAsync(generationConfiguration.OpenAPIFilePath, generationConfiguration, null, false, cancellationToken).ConfigureAwait(false); +#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task + await using var msForOpenAPIDocument = new MemoryStream(); // openapi.net doesn't honour leave open + await using var ms = new MemoryStream(); +#pragma warning restore CA2007 // Consider calling ConfigureAwait on the awaited task + await stream.CopyToAsync(msForOpenAPIDocument, cancellationToken).ConfigureAwait(false); + msForOpenAPIDocument.Seek(0, SeekOrigin.Begin); + await msForOpenAPIDocument.CopyToAsync(ms, cancellationToken).ConfigureAwait(false); + ms.Seek(0, SeekOrigin.Begin); + msForOpenAPIDocument.Seek(0, SeekOrigin.Begin); + var document = await openApiDocumentDownloadService.GetDocumentFromStreamAsync(msForOpenAPIDocument, generationConfiguration, false, cancellationToken).ConfigureAwait(false); + generationConfiguration.ApiRootUrl = document?.GetAPIRootUrl(generationConfiguration.OpenAPIFilePath); + await descriptionStorageService.UpdateDescriptionAsync(generationConfiguration.ClientClassName, ms, string.Empty, cancellationToken).ConfigureAwait(false); + + var clientConfiguration = new ApiClientConfiguration(generationConfiguration); + clientConfiguration.NormalizePaths(WorkingDirectory); + wsConfig.Clients.Add(generationConfiguration.ClientClassName, clientConfiguration); + var inputConfigurationHash = await GetConfigurationHashAsync(clientConfiguration, "migrated-pending-generate").ConfigureAwait(false); // because it's a migration, we don't want to calculate the exact hash since the description might have changed since the initial generation that created the lock file - apiManifest.ApiDependencies.Add(configuration.ClientClassName, configuration.ToApiDependency(inputConfigurationHash, new()));//TODO get the resolved operations? - var (stream, _) = await DownloadHelper.LoadStream(configuration.OpenAPIFilePath, HttpClient, Logger, configuration, null, false, cancellationToken).ConfigureAwait(false); - await descriptionStorageService.UpdateDescriptionAsync(configuration.ClientClassName, stream, string.Empty, cancellationToken).ConfigureAwait(false); - lockManagementService.DeleteLockFile(Path.GetDirectoryName(configuration.OpenAPIFilePath)!); + apiManifest.ApiDependencies.Add(generationConfiguration.ClientClassName, generationConfiguration.ToApiDependency(inputConfigurationHash, new())); + lockManagementService.DeleteLockFile(Path.GetDirectoryName(generationConfiguration.OpenAPIFilePath)!); } await workspaceConfigurationStorageService.UpdateWorkspaceConfigurationAsync(wsConfig, apiManifest, cancellationToken).ConfigureAwait(false); return clientsGenerationConfigurations.OfType().Select(static x => x.ClientClassName); From beca262c302d4b6706e0e8b808f293b8d2778160 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 12:04:26 -0500 Subject: [PATCH 11/17] - fixes lock file deletion Signed-off-by: Vincent Biret --- .../WorkspaceManagement/WorkspaceManagementService.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index efca606b81..505e99b7e8 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -245,8 +245,8 @@ public async Task> MigrateFromLockFileAsync(string clientNam wsConfig.Clients.Add(generationConfiguration.ClientClassName, clientConfiguration); var inputConfigurationHash = await GetConfigurationHashAsync(clientConfiguration, "migrated-pending-generate").ConfigureAwait(false); // because it's a migration, we don't want to calculate the exact hash since the description might have changed since the initial generation that created the lock file - apiManifest.ApiDependencies.Add(generationConfiguration.ClientClassName, generationConfiguration.ToApiDependency(inputConfigurationHash, new())); - lockManagementService.DeleteLockFile(Path.GetDirectoryName(generationConfiguration.OpenAPIFilePath)!); + apiManifest.ApiDependencies.Add(generationConfiguration.ClientClassName, generationConfiguration.ToApiDependency(inputConfigurationHash, [])); + lockManagementService.DeleteLockFile(Path.Combine(WorkingDirectory, clientConfiguration.OutputPath)); } await workspaceConfigurationStorageService.UpdateWorkspaceConfigurationAsync(wsConfig, apiManifest, cancellationToken).ConfigureAwait(false); return clientsGenerationConfigurations.OfType().Select(static x => x.ClientClassName); @@ -266,6 +266,7 @@ public async Task> MigrateFromLockFileAsync(string clientNam } var generationConfiguration = new GenerationConfiguration(); lockInfo.UpdateGenerationConfigurationFromLock(generationConfiguration); + generationConfiguration.OutputPath = "./" + Path.GetRelativePath(WorkingDirectory, lockFileDirectory); if (!string.IsNullOrEmpty(clientName)) { generationConfiguration.ClientClassName = clientName; From 2a09fdbca88d474bfeaeb58243773fb4a008201b Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 12:17:52 -0500 Subject: [PATCH 12/17] - updates message display for migrate command Signed-off-by: Vincent Biret --- src/kiota/Handlers/BaseKiotaCommandHandler.cs | 5 +++++ src/kiota/Handlers/Config/MigrateHandler.cs | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/kiota/Handlers/BaseKiotaCommandHandler.cs b/src/kiota/Handlers/BaseKiotaCommandHandler.cs index 8cc071ff86..32b0baef3a 100644 --- a/src/kiota/Handlers/BaseKiotaCommandHandler.cs +++ b/src/kiota/Handlers/BaseKiotaCommandHandler.cs @@ -243,6 +243,11 @@ _ when string.IsNullOrEmpty(version) => $"Example: kiota show -k {searchTerm} -- DisplayHint("Hint: use the --include-path and --exclude-path options with glob patterns to filter the paths displayed.", example); } } + protected void DisplayGenerateAfterMigrateHint() + { + DisplayHint("Hint: use the generate command to update the client and the manifest requests.", + "Example: kiota client generate"); + } protected void DisplaySearchAddHint() { DisplayHint("Hint: add your own API to the search result https://aka.ms/kiota/addapi."); diff --git a/src/kiota/Handlers/Config/MigrateHandler.cs b/src/kiota/Handlers/Config/MigrateHandler.cs index 1be224cfc8..cb84720b8b 100644 --- a/src/kiota/Handlers/Config/MigrateHandler.cs +++ b/src/kiota/Handlers/Config/MigrateHandler.cs @@ -28,7 +28,7 @@ public override async Task InvokeAsync(InvocationContext context) string clientName = context.ParseResult.GetValueForOption(ClassOption) ?? string.Empty; CancellationToken cancellationToken = context.BindingContext.GetService(typeof(CancellationToken)) is CancellationToken token ? token : CancellationToken.None; lockDirectory = NormalizeSlashesInPath(lockDirectory); - var (loggerFactory, logger) = GetLoggerAndFactory(context, Configuration.Generation.OutputPath); + var (loggerFactory, logger) = GetLoggerAndFactory(context, $"./{DescriptionStorageService.KiotaDirectorySegment}"); using (loggerFactory) { try @@ -37,10 +37,11 @@ public override async Task InvokeAsync(InvocationContext context) var clientNames = await workspaceManagementService.MigrateFromLockFileAsync(clientName, lockDirectory, cancellationToken).ConfigureAwait(false); if (!clientNames.Any()) { - logger.LogWarning("no client configuration was migrated"); + DisplayWarning("no client configuration was migrated"); return 1; } - logger.LogInformation("client configurations migrated successfully: {clientNames}", string.Join(", ", clientNames)); + DisplaySuccess($"Client configurations migrated successfully: {string.Join(", ", clientNames)}"); + DisplayGenerateAfterMigrateHint(); return 0; } catch (Exception ex) From a9006bc4d74658a3d878843241fd5891e0af9c24 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 12:18:42 -0500 Subject: [PATCH 13/17] - adds launch configuration for migrate Signed-off-by: Vincent Biret --- .vscode/launch.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 36465ec859..e5404b0746 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -265,6 +265,20 @@ "console": "internalConsole", "stopAtEntry": false }, + { + "name": "Launch Migrate", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + "program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll", + "args": ["config", "migrate"], + "cwd": "${workspaceFolder}/samples/msgraph-mail/dotnet", + "console": "internalConsole", + "stopAtEntry": false, + "env": { + "KIOTA_CONFIG_PREVIEW": "true" + } + }, { "name": "Launch Login (github - device)", "type": "coreclr", From 317cd2dbd09774f912d80cd6e4cbd0f17d25bcbb Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 12:22:34 -0500 Subject: [PATCH 14/17] - code linting Signed-off-by: Vincent Biret --- .../WorkspaceManagement/WorkspaceConfigurationStorageService.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs index 636d1e443e..02cb20ca51 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs @@ -1,6 +1,4 @@ using System; -using System.Collections; -using System.Collections.Generic; using System.IO; using System.Security.Cryptography; using System.Text; From 852ad09afd7ae629a9d92d6aa71eb2815a7a4bde Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 14:10:25 -0500 Subject: [PATCH 15/17] - adds tests for lock delete Signed-off-by: Vincent Biret --- .../Extensions/OpenApiDocumentExtensions.cs | 4 ++-- src/Kiota.Builder/KiotaBuilder.cs | 7 +++---- .../WorkspaceManagementService.cs | 8 +++++++- .../Lock/LockManagementServiceTests.cs | 16 ++++++++++++++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs b/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs index dcfdb31c5b..dc46ca6807 100644 --- a/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs +++ b/src/Kiota.Builder/Extensions/OpenApiDocumentExtensions.cs @@ -29,11 +29,11 @@ internal static void InitializeInheritanceIndex(this OpenApiDocument openApiDocu } internal static string? GetAPIRootUrl(this OpenApiDocument openApiDocument, string openAPIFilePath) { - if (openApiDocument == null) return null; + ArgumentNullException.ThrowIfNull(openApiDocument); var candidateUrl = openApiDocument.Servers .GroupBy(static x => x, new OpenApiServerComparer()) //group by protocol relative urls .FirstOrDefault() - ?.OrderByDescending(static x => x?.Url, StringComparer.OrdinalIgnoreCase) // prefer https over http + ?.OrderByDescending(static x => x.Url, StringComparer.OrdinalIgnoreCase) // prefer https over http ?.FirstOrDefault() ?.Url; if (string.IsNullOrEmpty(candidateUrl)) diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index d150c807e0..42b40b2d03 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -355,11 +355,10 @@ internal void FilterPathsByPatterns(OpenApiDocument doc) } internal void SetApiRootUrl() { - var candidateUrl = openApiDocument?.GetAPIRootUrl(config.OpenAPIFilePath); - if (candidateUrl is null) - logger.LogWarning("No server url found in the OpenAPI document. The base url will need to be set when using the client."); - else + if (openApiDocument is not null && openApiDocument.GetAPIRootUrl(config.OpenAPIFilePath) is string candidateUrl) config.ApiRootUrl = candidateUrl; + else + logger.LogWarning("No server url found in the OpenAPI document. The base url will need to be set when using the client."); } private void StopLogAndReset(Stopwatch sw, string prefix) { diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index 505e99b7e8..f675f469af 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -237,7 +237,13 @@ public async Task> MigrateFromLockFileAsync(string clientNam ms.Seek(0, SeekOrigin.Begin); msForOpenAPIDocument.Seek(0, SeekOrigin.Begin); var document = await openApiDocumentDownloadService.GetDocumentFromStreamAsync(msForOpenAPIDocument, generationConfiguration, false, cancellationToken).ConfigureAwait(false); - generationConfiguration.ApiRootUrl = document?.GetAPIRootUrl(generationConfiguration.OpenAPIFilePath); + if (document is null) + { + Logger.LogError("The client {ClientName} could not be migrated because the OpenAPI document could not be loaded", generationConfiguration.ClientClassName); + clientsGenerationConfigurations.Remove(generationConfiguration); + continue; + } + generationConfiguration.ApiRootUrl = document.GetAPIRootUrl(generationConfiguration.OpenAPIFilePath); await descriptionStorageService.UpdateDescriptionAsync(generationConfiguration.ClientClassName, ms, string.Empty, cancellationToken).ConfigureAwait(false); var clientConfiguration = new ApiClientConfiguration(generationConfiguration); diff --git a/tests/Kiota.Builder.Tests/Lock/LockManagementServiceTests.cs b/tests/Kiota.Builder.Tests/Lock/LockManagementServiceTests.cs index df6c5365ab..a00aafe3ff 100644 --- a/tests/Kiota.Builder.Tests/Lock/LockManagementServiceTests.cs +++ b/tests/Kiota.Builder.Tests/Lock/LockManagementServiceTests.cs @@ -52,4 +52,20 @@ public async Task UsesRelativePaths() await lockManagementService.WriteLockFileAsync(outputDirectory, lockFile); Assert.Equal($"..{Path.DirectorySeparatorChar}information{Path.DirectorySeparatorChar}description.yml", lockFile.DescriptionLocation, StringComparer.OrdinalIgnoreCase); } + [Fact] + public async Task DeletesALock() + { + var lockManagementService = new LockManagementService(); + var descriptionPath = Path.Combine(Path.GetTempPath(), "description.yml"); + var lockFile = new KiotaLock + { + ClientClassName = "foo", + ClientNamespaceName = "bar", + DescriptionLocation = descriptionPath, + }; + var path = Path.GetTempPath(); + await lockManagementService.WriteLockFileAsync(path, lockFile); + lockManagementService.DeleteLockFile(path); + Assert.Null(await lockManagementService.GetLockFromDirectoryAsync(path)); + } } From 3f3437b1dcca7e86b7eaf71c1ee84461fa3a8366 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 1 Mar 2024 14:52:56 -0500 Subject: [PATCH 16/17] - adds unit tests for migration orchestration Signed-off-by: Vincent Biret --- .../ApiClientConfiguration.cs | 3 +- .../DescriptionStorageService.cs | 2 +- .../WorkspaceManagementService.cs | 2 +- .../WorkspaceManagementServiceTests.cs | 85 +++++++++++++++++++ 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/Kiota.Builder/WorkspaceManagement/ApiClientConfiguration.cs b/src/Kiota.Builder/WorkspaceManagement/ApiClientConfiguration.cs index cf206d7437..46a3f98167 100644 --- a/src/Kiota.Builder/WorkspaceManagement/ApiClientConfiguration.cs +++ b/src/Kiota.Builder/WorkspaceManagement/ApiClientConfiguration.cs @@ -158,7 +158,8 @@ public object Clone() } public void NormalizePaths(string targetDirectory) { - OutputPath = "./" + Path.GetRelativePath(targetDirectory, OutputPath); + if (Path.IsPathRooted(OutputPath)) + OutputPath = "./" + Path.GetRelativePath(targetDirectory, OutputPath); } } #pragma warning restore CA2227 // Collection properties should be read only diff --git a/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs b/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs index 6facf131b6..1bdd930b69 100644 --- a/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs @@ -9,7 +9,7 @@ namespace Kiota.Builder.WorkspaceManagement; public class DescriptionStorageService { public const string KiotaDirectorySegment = ".kiota"; - private const string DescriptionsSubDirectoryRelativePath = $"{KiotaDirectorySegment}/clients"; + internal const string DescriptionsSubDirectoryRelativePath = $"{KiotaDirectorySegment}/clients"; private readonly string TargetDirectory; public DescriptionStorageService(string targetDirectory) { diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index f675f469af..8ea6d11970 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -244,7 +244,7 @@ public async Task> MigrateFromLockFileAsync(string clientNam continue; } generationConfiguration.ApiRootUrl = document.GetAPIRootUrl(generationConfiguration.OpenAPIFilePath); - await descriptionStorageService.UpdateDescriptionAsync(generationConfiguration.ClientClassName, ms, string.Empty, cancellationToken).ConfigureAwait(false); + await descriptionStorageService.UpdateDescriptionAsync(generationConfiguration.ClientClassName, ms, new Uri(generationConfiguration.OpenAPIFilePath).GetFileExtension(), cancellationToken).ConfigureAwait(false); var clientConfiguration = new ApiClientConfiguration(generationConfiguration); clientConfiguration.NormalizePaths(WorkingDirectory); diff --git a/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs b/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs index cfdb3e0ac2..f02053143d 100644 --- a/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs +++ b/tests/Kiota.Builder.Tests/WorkspaceManagement/WorkspaceManagementServiceTests.cs @@ -1,8 +1,10 @@ using System; using System.IO; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; using Kiota.Builder.Configuration; +using Kiota.Builder.Lock; using Kiota.Builder.WorkspaceManagement; using Microsoft.Extensions.Logging; using Moq; @@ -87,6 +89,89 @@ public async Task RemovesAClient() var result = await service.IsClientPresent("clientName"); Assert.False(result); } + [Fact] + public async Task FailsOnMigrateWithoutKiotaConfigMode() + { + var mockLogger = Mock.Of(); + Directory.CreateDirectory(tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, false, tempPath); + await Assert.ThrowsAsync(() => service.MigrateFromLockFileAsync(string.Empty, tempPath)); + } + [Fact] + public async Task FailsWhenTargetLockDirectoryIsNotSubDirectory() + { + var mockLogger = Mock.Of(); + Directory.CreateDirectory(tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, true, tempPath); + await Assert.ThrowsAsync(() => service.MigrateFromLockFileAsync(string.Empty, Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()))); + } + [Fact] + public async Task FailsWhenNoLockFilesAreFound() + { + var mockLogger = Mock.Of(); + Directory.CreateDirectory(tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, true, tempPath); + await Assert.ThrowsAsync(() => service.MigrateFromLockFileAsync(string.Empty, tempPath)); + } + [Fact] + public async Task FailsOnMultipleLockFilesAndClientName() + { + var mockLogger = Mock.Of(); + Directory.CreateDirectory(tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, true, tempPath); + Directory.CreateDirectory(Path.Combine(tempPath, "client1")); + Directory.CreateDirectory(Path.Combine(tempPath, "client2")); + File.WriteAllText(Path.Combine(tempPath, "client1", LockManagementService.LockFileName), "foo"); + File.WriteAllText(Path.Combine(tempPath, "client2", LockManagementService.LockFileName), "foo"); + await Assert.ThrowsAsync(() => service.MigrateFromLockFileAsync("bar", tempPath)); + } + [Fact] + public async Task MigratesAClient() + { + var mockLogger = Mock.Of(); + Directory.CreateDirectory(tempPath); + var service = new WorkspaceManagementService(mockLogger, httpClient, true, tempPath); + var descriptionPath = Path.Combine(tempPath, "description.yml"); + var generationConfiguration = new GenerationConfiguration + { + ClientClassName = "clientName", + OutputPath = Path.Combine(tempPath, "client"), + OpenAPIFilePath = descriptionPath, + ApiRootUrl = "https://graph.microsoft.com", + }; + Directory.CreateDirectory(generationConfiguration.OutputPath); + await File.WriteAllTextAsync(descriptionPath, @$"openapi: 3.0.1 +info: + title: OData Service for namespace microsoft.graph + description: This OData service is located at https://graph.microsoft.com/v1.0 + version: 1.0.1 +servers: + - url: https://localhost:443 +paths: + /enumeration: + get: + responses: + '200': + content: + application/json: + schema: + type: object + properties: + bar: + type: object + properties: + foo: + type: string"); + var classicService = new WorkspaceManagementService(mockLogger, httpClient, false, tempPath); + await classicService.UpdateStateFromConfigurationAsync(generationConfiguration, "foo", [], Stream.Null); + var clientNames = await service.MigrateFromLockFileAsync("clientName", tempPath); + Assert.Single(clientNames); + Assert.Equal("clientName", clientNames.First()); + Assert.False(File.Exists(Path.Combine(tempPath, LockManagementService.LockFileName))); + Assert.True(File.Exists(Path.Combine(tempPath, WorkspaceConfigurationStorageService.ConfigurationFileName))); + Assert.True(File.Exists(Path.Combine(tempPath, WorkspaceConfigurationStorageService.ManifestFileName))); + Assert.True(File.Exists(Path.Combine(tempPath, DescriptionStorageService.DescriptionsSubDirectoryRelativePath, "clientName.yml"))); + } public void Dispose() { From 06828dbc7f874a1c34bfd8b4d27ee08738234ce9 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Tue, 5 Mar 2024 13:32:35 -0500 Subject: [PATCH 17/17] - removes unnecessary field Signed-off-by: Vincent Biret --- .../WorkspaceManagement/WorkspaceManagementService.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs index 8ea6d11970..7874db312c 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceManagementService.cs @@ -24,20 +24,18 @@ public class WorkspaceManagementService { private readonly bool UseKiotaConfig; private readonly ILogger Logger; - private readonly HttpClient HttpClient; public WorkspaceManagementService(ILogger logger, HttpClient httpClient, bool useKiotaConfig = false, string workingDirectory = "") { ArgumentNullException.ThrowIfNull(logger); ArgumentNullException.ThrowIfNull(httpClient); Logger = logger; - HttpClient = httpClient; UseKiotaConfig = useKiotaConfig; if (string.IsNullOrEmpty(workingDirectory)) workingDirectory = Directory.GetCurrentDirectory(); WorkingDirectory = workingDirectory; workspaceConfigurationStorageService = new(workingDirectory); descriptionStorageService = new(workingDirectory); - openApiDocumentDownloadService = new(HttpClient, Logger); + openApiDocumentDownloadService = new(httpClient, Logger); } private readonly OpenApiDocumentDownloadService openApiDocumentDownloadService; private readonly LockManagementService lockManagementService = new();