From cb1479ad26b4f7ea6788934bb865a45275bc9878 Mon Sep 17 00:00:00 2001 From: Vincent Biret Date: Fri, 23 Feb 2024 14:59:43 -0500 Subject: [PATCH] - adds workspace save of description Signed-off-by: Vincent Biret --- src/Kiota.Builder/Extensions/UriExtensions.cs | 5 ++ src/Kiota.Builder/KiotaBuilder.cs | 47 +++++++--- .../DescriptionStorageService.cs | 53 ++++++++++++ .../WorkspaceConfigurationStorageService.cs | 86 +++++++++++-------- 4 files changed, 146 insertions(+), 45 deletions(-) create mode 100644 src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs diff --git a/src/Kiota.Builder/Extensions/UriExtensions.cs b/src/Kiota.Builder/Extensions/UriExtensions.cs index b0c8e235d5..938f269d9b 100644 --- a/src/Kiota.Builder/Extensions/UriExtensions.cs +++ b/src/Kiota.Builder/Extensions/UriExtensions.cs @@ -10,4 +10,9 @@ public static string GetFileName(this Uri uri) if (uri is null) return string.Empty; return Path.GetFileName($"{uri.Scheme}://{uri.Host}{uri.AbsolutePath}"); } + public static string GetFileExtension(this Uri uri) + { + if (uri is null) return string.Empty; + return Path.GetExtension(uri.GetFileName()).TrimStart('.'); + } } diff --git a/src/Kiota.Builder/KiotaBuilder.cs b/src/Kiota.Builder/KiotaBuilder.cs index da565895f9..abca8d28f4 100644 --- a/src/Kiota.Builder/KiotaBuilder.cs +++ b/src/Kiota.Builder/KiotaBuilder.cs @@ -63,8 +63,12 @@ public KiotaBuilder(ILogger logger, GenerationConfiguration config { MaxDegreeOfParallelism = config.MaxDegreeOfParallelism, }; - workspaceManagementService = new WorkspaceManagementService(logger, useKiotaConfig); + var workingDirectory = Directory.GetCurrentDirectory(); + workspaceManagementService = new WorkspaceManagementService(logger, useKiotaConfig, workingDirectory); + descriptionStorageService = new DescriptionStorageService(workingDirectory); + this.useKiotaConfig = useKiotaConfig; } + private readonly bool useKiotaConfig; private async Task CleanOutputDirectory(CancellationToken cancellationToken) { if (config.CleanOutput && Directory.Exists(config.OutputPath)) @@ -268,20 +272,14 @@ public async Task GenerateClientAsync(CancellationToken cancellationToken) await CreateLanguageSourceFilesAsync(config.Language, generatedCode, cancellationToken).ConfigureAwait(false); StopLogAndReset(sw, $"step {++stepId} - writing files - took"); - // Write lock file - sw.Start(); - await workspaceManagementService.UpdateStateFromConfigurationAsync(config, openApiDocument?.HashCode ?? string.Empty, openApiTree?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], cancellationToken).ConfigureAwait(false); - StopLogAndReset(sw, $"step {++stepId} - writing lock file - took"); + await FinalizeWorkspaceAsync(sw, stepId, openApiTree, inputPath, cancellationToken).ConfigureAwait(false); } else { logger.LogInformation("No changes detected, skipping generation"); if (config.Operation is ClientOperation.Add or ClientOperation.Edit && config.SkipGeneration) { - // Write lock file - sw.Start(); - await workspaceManagementService.UpdateStateFromConfigurationAsync(config, openApiDocument?.HashCode ?? string.Empty, openApiTree?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], cancellationToken).ConfigureAwait(false); - StopLogAndReset(sw, $"step {++stepId} - writing lock file - took"); + await FinalizeWorkspaceAsync(sw, stepId, openApiTree, inputPath, cancellationToken).ConfigureAwait(false); } return false; } @@ -293,6 +291,22 @@ public async Task GenerateClientAsync(CancellationToken cancellationToken) } return true; } + private async Task FinalizeWorkspaceAsync(Stopwatch sw, int stepId, OpenApiUrlTreeNode? openApiTree, string inputPath, CancellationToken cancellationToken) + { + // Write lock file + sw.Start(); + await workspaceManagementService.UpdateStateFromConfigurationAsync(config, openApiDocument?.HashCode ?? string.Empty, openApiTree?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], cancellationToken).ConfigureAwait(false); + StopLogAndReset(sw, $"step {++stepId} - writing lock file - took"); + + if (!isDescriptionFromWorkspaceCopy) + { + // Store description in the workspace copy + sw.Start(); + using var descriptionStream = await LoadStream(inputPath, cancellationToken).ConfigureAwait(false); + await descriptionStorageService.UpdateDescriptionAsync(config.ClientClassName, descriptionStream, new Uri(config.OpenAPIFilePath).GetFileExtension(), cancellationToken).ConfigureAwait(false); + StopLogAndReset(sw, $"step {++stepId} - storing description in the workspace copy - took"); + } + } private readonly WorkspaceManagementService workspaceManagementService; private static readonly GlobComparer globComparer = new(); [GeneratedRegex(@"([\/\\])\{[\w\d-]+\}([\/\\])", RegexOptions.IgnoreCase | RegexOptions.Singleline, 2000)] @@ -394,7 +408,8 @@ private void StopLogAndReset(Stopwatch sw, string prefix) o.PoolSize = 20; o.PoolInitialFill = 1; }); - + private readonly DescriptionStorageService descriptionStorageService; + private bool isDescriptionFromWorkspaceCopy; private async Task LoadStream(string inputPath, CancellationToken cancellationToken) { var stopwatch = new Stopwatch(); @@ -403,7 +418,15 @@ private async Task LoadStream(string inputPath, CancellationToken cancel inputPath = inputPath.Trim(); Stream input; - if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) + if (useKiotaConfig && + config.Operation is ClientOperation.Edit or ClientOperation.Add && + await descriptionStorageService.GetDescriptionAsync(config.ClientClassName, new Uri(inputPath).GetFileExtension(), cancellationToken).ConfigureAwait(false) is { } descriptionStream) + { + logger.LogInformation("loaded description from the workspace copy"); + input = descriptionStream; + isDescriptionFromWorkspaceCopy = true; + } + else if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase)) try { var cachingProvider = new DocumentCachingProvider(httpClient, logger) @@ -413,6 +436,7 @@ private async Task LoadStream(string inputPath, CancellationToken cancel var targetUri = APIsGuruSearchProvider.ChangeSourceUrlToGitHub(new Uri(inputPath)); // so updating existing clients doesn't break var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml"; input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false); + logger.LogInformation("loaded description from remote source"); } catch (HttpRequestException ex) { @@ -430,6 +454,7 @@ private async Task LoadStream(string inputPath, CancellationToken cancel } inMemoryStream.Position = 0; input = inMemoryStream; + logger.LogInformation("loaded description from local source"); #pragma warning restore CA2000 } catch (Exception ex) when (ex is FileNotFoundException || diff --git a/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs b/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs new file mode 100644 index 0000000000..a024b61962 --- /dev/null +++ b/src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using AsyncKeyedLock; + +namespace Kiota.Builder.WorkspaceManagement; + +public class DescriptionStorageService +{ + private const string DescriptionsSubDirectoryRelativePath = ".kiota/clients"; + private readonly string TargetDirectory; + public DescriptionStorageService(string targetDirectory) + { + ArgumentException.ThrowIfNullOrEmpty(targetDirectory); + TargetDirectory = targetDirectory; + } + private static readonly AsyncKeyedLocker localFilesLock = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); + public async Task UpdateDescriptionAsync(string clientName, Stream description, string extension = "yml", CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(clientName); + ArgumentNullException.ThrowIfNull(description); + ArgumentNullException.ThrowIfNull(extension); + var descriptionFilePath = Path.Combine(TargetDirectory, DescriptionsSubDirectoryRelativePath, $"{clientName}.{extension}"); + using (await localFilesLock.LockAsync(descriptionFilePath, cancellationToken).ConfigureAwait(false)) + { + Directory.CreateDirectory(Path.GetDirectoryName(descriptionFilePath) ?? throw new InvalidOperationException("The target path is invalid")); + using var fs = new FileStream(descriptionFilePath, FileMode.Create); + description.Seek(0, SeekOrigin.Begin); + await description.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); + } + } + public async Task GetDescriptionAsync(string clientName, string extension = "yml", CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(clientName); + ArgumentNullException.ThrowIfNull(extension); + var descriptionFilePath = Path.Combine(TargetDirectory, DescriptionsSubDirectoryRelativePath, $"{clientName}.{extension}"); + if (!File.Exists(descriptionFilePath)) + return null; + using (await localFilesLock.LockAsync(descriptionFilePath, cancellationToken).ConfigureAwait(false)) + { + using var fs = new FileStream(descriptionFilePath, FileMode.Open); + var ms = new MemoryStream(); + await fs.CopyToAsync(ms, cancellationToken).ConfigureAwait(false); + ms.Seek(0, SeekOrigin.Begin); + return ms; + } + } +} diff --git a/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs b/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs index 00e3be7959..158eebe4b6 100644 --- a/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs +++ b/src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs @@ -5,6 +5,7 @@ using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using AsyncKeyedLock; using Kiota.Builder.Manifest; using Microsoft.OpenApi.ApiManifest; @@ -28,6 +29,11 @@ public WorkspaceConfigurationStorageService(string targetDirectory) targetConfigurationFilePath = Path.Combine(TargetDirectory, ConfigurationFileName); targetManifestFilePath = Path.Combine(TargetDirectory, ManifestFileName); } + private static readonly AsyncKeyedLocker localFilesLock = new(o => + { + o.PoolSize = 20; + o.PoolInitialFill = 1; + }); public async Task InitializeAsync(CancellationToken cancellationToken = default) { if (await IsInitializedAsync(cancellationToken).ConfigureAwait(false)) @@ -37,18 +43,24 @@ public async Task InitializeAsync(CancellationToken cancellationToken = default) public async Task UpdateWorkspaceConfigurationAsync(WorkspaceConfiguration configuration, ApiManifestDocument? manifestDocument, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(configuration); - if (!Directory.Exists(TargetDirectory)) - Directory.CreateDirectory(TargetDirectory); + using (await localFilesLock.LockAsync(targetConfigurationFilePath, cancellationToken).ConfigureAwait(false)) + { + if (!Directory.Exists(TargetDirectory)) + Directory.CreateDirectory(TargetDirectory); #pragma warning disable CA2007 - await using var configStream = File.Open(targetConfigurationFilePath, FileMode.Create); + await using var configStream = File.Open(targetConfigurationFilePath, FileMode.Create); #pragma warning restore CA2007 - await JsonSerializer.SerializeAsync(configStream, configuration, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false); - if (manifestDocument != null) - { + await JsonSerializer.SerializeAsync(configStream, configuration, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false); + if (manifestDocument != null) + { + using (await localFilesLock.LockAsync(targetManifestFilePath, cancellationToken).ConfigureAwait(false)) + { #pragma warning disable CA2007 - await using var manifestStream = File.Open(targetManifestFilePath, FileMode.Create); + await using var manifestStream = File.Open(targetManifestFilePath, FileMode.Create); #pragma warning restore CA2007 - await manifestManagementService.SerializeManifestDocumentAsync(manifestDocument, manifestStream).ConfigureAwait(false); + await manifestManagementService.SerializeManifestDocumentAsync(manifestDocument, manifestStream).ConfigureAwait(false); + } + } } } public Task IsInitializedAsync(CancellationToken cancellationToken = default) @@ -64,51 +76,54 @@ public Task IsInitializedAsync(CancellationToken cancellationToken = defau public async Task<(WorkspaceConfiguration?, ApiManifestDocument?)> GetWorkspaceConfigurationAsync(CancellationToken cancellationToken = default) { if (File.Exists(targetConfigurationFilePath)) - { + using (await localFilesLock.LockAsync(targetConfigurationFilePath, cancellationToken).ConfigureAwait(false)) + { #pragma warning disable CA2007 - await using var configStream = File.OpenRead(targetConfigurationFilePath); + await using var configStream = File.OpenRead(targetConfigurationFilePath); #pragma warning restore CA2007 - var config = await JsonSerializer.DeserializeAsync(configStream, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false); - if (File.Exists(targetManifestFilePath)) - { + var config = await JsonSerializer.DeserializeAsync(configStream, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false); + if (File.Exists(targetManifestFilePath)) + using (await localFilesLock.LockAsync(targetManifestFilePath, cancellationToken).ConfigureAwait(false)) + { #pragma warning disable CA2007 - await using var manifestStream = File.OpenRead(targetManifestFilePath); + await using var manifestStream = File.OpenRead(targetManifestFilePath); #pragma warning restore CA2007 - var manifest = await manifestManagementService.DeserializeManifestDocumentAsync(manifestStream).ConfigureAwait(false); - return (config, manifest); + var manifest = await manifestManagementService.DeserializeManifestDocumentAsync(manifestStream).ConfigureAwait(false); + return (config, manifest); + } + return (config, null); } - return (config, null); - } return (null, null); } - public Task BackupConfigAsync(string directoryPath, CancellationToken cancellationToken = default) + public async Task BackupConfigAsync(string directoryPath, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrEmpty(directoryPath); - BackupFile(directoryPath, ConfigurationFileName); - BackupFile(directoryPath, ManifestFileName); - return Task.CompletedTask; + await BackupFile(directoryPath, ConfigurationFileName, cancellationToken).ConfigureAwait(false); + await BackupFile(directoryPath, ManifestFileName, cancellationToken).ConfigureAwait(false); } - private static void BackupFile(string directoryPath, string fileName) + private static async Task BackupFile(string directoryPath, string fileName, CancellationToken cancellationToken = default) { var sourceFilePath = Path.Combine(directoryPath, fileName); if (File.Exists(sourceFilePath)) { var backupFilePath = GetBackupFilePath(directoryPath, fileName); - var targetDirectory = Path.GetDirectoryName(backupFilePath); - if (string.IsNullOrEmpty(targetDirectory)) return; - if (!Directory.Exists(targetDirectory)) - Directory.CreateDirectory(targetDirectory); - File.Copy(sourceFilePath, backupFilePath, true); + using (await localFilesLock.LockAsync(backupFilePath, cancellationToken).ConfigureAwait(false)) + { + var targetDirectory = Path.GetDirectoryName(backupFilePath); + if (string.IsNullOrEmpty(targetDirectory)) return; + if (!Directory.Exists(targetDirectory)) + Directory.CreateDirectory(targetDirectory); + File.Copy(sourceFilePath, backupFilePath, true); + } } } - public Task RestoreConfigAsync(string directoryPath, CancellationToken cancellationToken = default) + public async Task RestoreConfigAsync(string directoryPath, CancellationToken cancellationToken = default) { ArgumentException.ThrowIfNullOrEmpty(directoryPath); - RestoreFile(directoryPath, ConfigurationFileName); - RestoreFile(directoryPath, ManifestFileName); - return Task.CompletedTask; + await RestoreFile(directoryPath, ConfigurationFileName, cancellationToken).ConfigureAwait(false); + await RestoreFile(directoryPath, ManifestFileName, cancellationToken).ConfigureAwait(false); } - private static void RestoreFile(string directoryPath, string fileName) + private static async Task RestoreFile(string directoryPath, string fileName, CancellationToken cancellationToken = default) { var sourceFilePath = Path.Combine(directoryPath, fileName); var targetDirectory = Path.GetDirectoryName(sourceFilePath); @@ -118,7 +133,10 @@ private static void RestoreFile(string directoryPath, string fileName) var backupFilePath = GetBackupFilePath(directoryPath, fileName); if (File.Exists(backupFilePath)) { - File.Copy(backupFilePath, sourceFilePath, true); + using (await localFilesLock.LockAsync(sourceFilePath, cancellationToken).ConfigureAwait(false)) + { + File.Copy(backupFilePath, sourceFilePath, true); + } } } private static readonly ThreadLocal HashAlgorithm = new(SHA256.Create);