- adds workspace save of description

Signed-off-by: Vincent Biret <[email protected]>

src/Kiota.Builder/Extensions/UriExtensions.cs

@@ -10,4 +10,9 @@ public static string GetFileName(this Uri uri)
         if (uri is null) return string.Empty;
         return Path.GetFileName($"{uri.Scheme}://{uri.Host}{uri.AbsolutePath}");
     }
+    public static string GetFileExtension(this Uri uri)
+    {
+        if (uri is null) return string.Empty;
+        return Path.GetExtension(uri.GetFileName()).TrimStart('.');
+    }
 }

src/Kiota.Builder/KiotaBuilder.cs

@@ -63,8 +63,12 @@ public KiotaBuilder(ILogger<KiotaBuilder> logger, GenerationConfiguration config
         {
             MaxDegreeOfParallelism = config.MaxDegreeOfParallelism,
         };
-        workspaceManagementService = new WorkspaceManagementService(logger, useKiotaConfig);
+        var workingDirectory = Directory.GetCurrentDirectory();
+        workspaceManagementService = new WorkspaceManagementService(logger, useKiotaConfig, workingDirectory);
+        descriptionStorageService = new DescriptionStorageService(workingDirectory);
+        this.useKiotaConfig = useKiotaConfig;
     }
+    private readonly bool useKiotaConfig;
     private async Task CleanOutputDirectory(CancellationToken cancellationToken)
     {
         if (config.CleanOutput && Directory.Exists(config.OutputPath))
@@ -268,20 +272,14 @@ public async Task<bool> GenerateClientAsync(CancellationToken cancellationToken)
                 await CreateLanguageSourceFilesAsync(config.Language, generatedCode, cancellationToken).ConfigureAwait(false);
                 StopLogAndReset(sw, $"step {++stepId} - writing files - took");
 
-                // Write lock file
-                sw.Start();
-                await workspaceManagementService.UpdateStateFromConfigurationAsync(config, openApiDocument?.HashCode ?? string.Empty, openApiTree?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], cancellationToken).ConfigureAwait(false);
-                StopLogAndReset(sw, $"step {++stepId} - writing lock file - took");
+                await FinalizeWorkspaceAsync(sw, stepId, openApiTree, inputPath, cancellationToken).ConfigureAwait(false);
             }
             else
             {
                 logger.LogInformation("No changes detected, skipping generation");
                 if (config.Operation is ClientOperation.Add or ClientOperation.Edit && config.SkipGeneration)
                 {
-                    // Write lock file
-                    sw.Start();
-                    await workspaceManagementService.UpdateStateFromConfigurationAsync(config, openApiDocument?.HashCode ?? string.Empty, openApiTree?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], cancellationToken).ConfigureAwait(false);
-                    StopLogAndReset(sw, $"step {++stepId} - writing lock file - took");
+                    await FinalizeWorkspaceAsync(sw, stepId, openApiTree, inputPath, cancellationToken).ConfigureAwait(false);
                 }
                 return false;
             }
@@ -293,6 +291,22 @@ public async Task<bool> GenerateClientAsync(CancellationToken cancellationToken)
         }
         return true;
     }
+    private async Task FinalizeWorkspaceAsync(Stopwatch sw, int stepId, OpenApiUrlTreeNode? openApiTree, string inputPath, CancellationToken cancellationToken)
+    {
+        // Write lock file
+        sw.Start();
+        await workspaceManagementService.UpdateStateFromConfigurationAsync(config, openApiDocument?.HashCode ?? string.Empty, openApiTree?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], cancellationToken).ConfigureAwait(false);
+        StopLogAndReset(sw, $"step {++stepId} - writing lock file - took");
+
+        if (!isDescriptionFromWorkspaceCopy)
+        {
+            // Store description in the workspace copy
+            sw.Start();
+            using var descriptionStream = await LoadStream(inputPath, cancellationToken).ConfigureAwait(false);
+            await descriptionStorageService.UpdateDescriptionAsync(config.ClientClassName, descriptionStream, new Uri(config.OpenAPIFilePath).GetFileExtension(), cancellationToken).ConfigureAwait(false);
+            StopLogAndReset(sw, $"step {++stepId} - storing description in the workspace copy - took");
+        }
+    }
     private readonly WorkspaceManagementService workspaceManagementService;
     private static readonly GlobComparer globComparer = new();
     [GeneratedRegex(@"([\/\\])\{[\w\d-]+\}([\/\\])", RegexOptions.IgnoreCase | RegexOptions.Singleline, 2000)]
@@ -394,7 +408,8 @@ private void StopLogAndReset(Stopwatch sw, string prefix)
         o.PoolSize = 20;
         o.PoolInitialFill = 1;
     });
-
+    private readonly DescriptionStorageService descriptionStorageService;
+    private bool isDescriptionFromWorkspaceCopy;
     private async Task<Stream> LoadStream(string inputPath, CancellationToken cancellationToken)
     {
         var stopwatch = new Stopwatch();
@@ -403,7 +418,15 @@ private async Task<Stream> LoadStream(string inputPath, CancellationToken cancel
         inputPath = inputPath.Trim();
 
         Stream input;
-        if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+        if (useKiotaConfig &&
+            config.Operation is ClientOperation.Edit or ClientOperation.Add &&
+            await descriptionStorageService.GetDescriptionAsync(config.ClientClassName, new Uri(inputPath).GetFileExtension(), cancellationToken).ConfigureAwait(false) is { } descriptionStream)
+        {
+            logger.LogInformation("loaded description from the workspace copy");
+            input = descriptionStream;
+            isDescriptionFromWorkspaceCopy = true;
+        }
+        else if (inputPath.StartsWith("http", StringComparison.OrdinalIgnoreCase))
             try
             {
                 var cachingProvider = new DocumentCachingProvider(httpClient, logger)
@@ -413,6 +436,7 @@ private async Task<Stream> LoadStream(string inputPath, CancellationToken cancel
                 var targetUri = APIsGuruSearchProvider.ChangeSourceUrlToGitHub(new Uri(inputPath)); // so updating existing clients doesn't break
                 var fileName = targetUri.GetFileName() is string name && !string.IsNullOrEmpty(name) ? name : "description.yml";
                 input = await cachingProvider.GetDocumentAsync(targetUri, "generation", fileName, cancellationToken: cancellationToken).ConfigureAwait(false);
+                logger.LogInformation("loaded description from remote source");
             }
             catch (HttpRequestException ex)
             {
@@ -430,6 +454,7 @@ private async Task<Stream> LoadStream(string inputPath, CancellationToken cancel
                 }
                 inMemoryStream.Position = 0;
                 input = inMemoryStream;
+                logger.LogInformation("loaded description from local source");
 #pragma warning restore CA2000
             }
             catch (Exception ex) when (ex is FileNotFoundException ||

src/Kiota.Builder/WorkspaceManagement/DescriptionStorageService.cs

@@ -0,0 +1,53 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using AsyncKeyedLock;
+
+namespace Kiota.Builder.WorkspaceManagement;
+
+public class DescriptionStorageService
+{
+    private const string DescriptionsSubDirectoryRelativePath = ".kiota/clients";
+    private readonly string TargetDirectory;
+    public DescriptionStorageService(string targetDirectory)
+    {
+        ArgumentException.ThrowIfNullOrEmpty(targetDirectory);
+        TargetDirectory = targetDirectory;
+    }
+    private static readonly AsyncKeyedLocker<string> localFilesLock = new(o =>
+    {
+        o.PoolSize = 20;
+        o.PoolInitialFill = 1;
+    });
+    public async Task UpdateDescriptionAsync(string clientName, Stream description, string extension = "yml", CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(clientName);
+        ArgumentNullException.ThrowIfNull(description);
+        ArgumentNullException.ThrowIfNull(extension);
+        var descriptionFilePath = Path.Combine(TargetDirectory, DescriptionsSubDirectoryRelativePath, $"{clientName}.{extension}");
+        using (await localFilesLock.LockAsync(descriptionFilePath, cancellationToken).ConfigureAwait(false))
+        {
+            Directory.CreateDirectory(Path.GetDirectoryName(descriptionFilePath) ?? throw new InvalidOperationException("The target path is invalid"));
+            using var fs = new FileStream(descriptionFilePath, FileMode.Create);
+            description.Seek(0, SeekOrigin.Begin);
+            await description.CopyToAsync(fs, cancellationToken).ConfigureAwait(false);
+        }
+    }
+    public async Task<Stream?> GetDescriptionAsync(string clientName, string extension = "yml", CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(clientName);
+        ArgumentNullException.ThrowIfNull(extension);
+        var descriptionFilePath = Path.Combine(TargetDirectory, DescriptionsSubDirectoryRelativePath, $"{clientName}.{extension}");
+        if (!File.Exists(descriptionFilePath))
+            return null;
+        using (await localFilesLock.LockAsync(descriptionFilePath, cancellationToken).ConfigureAwait(false))
+        {
+            using var fs = new FileStream(descriptionFilePath, FileMode.Open);
+            var ms = new MemoryStream();
+            await fs.CopyToAsync(ms, cancellationToken).ConfigureAwait(false);
+            ms.Seek(0, SeekOrigin.Begin);
+            return ms;
+        }
+    }
+}

src/Kiota.Builder/WorkspaceManagement/WorkspaceConfigurationStorageService.cs

@@ -5,6 +5,7 @@
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
+using AsyncKeyedLock;
 using Kiota.Builder.Manifest;
 using Microsoft.OpenApi.ApiManifest;
 
@@ -28,6 +29,11 @@ public WorkspaceConfigurationStorageService(string targetDirectory)
         targetConfigurationFilePath = Path.Combine(TargetDirectory, ConfigurationFileName);
         targetManifestFilePath = Path.Combine(TargetDirectory, ManifestFileName);
     }
+    private static readonly AsyncKeyedLocker<string> localFilesLock = new(o =>
+    {
+        o.PoolSize = 20;
+        o.PoolInitialFill = 1;
+    });
     public async Task InitializeAsync(CancellationToken cancellationToken = default)
     {
         if (await IsInitializedAsync(cancellationToken).ConfigureAwait(false))
@@ -37,18 +43,24 @@ public async Task InitializeAsync(CancellationToken cancellationToken = default)
     public async Task UpdateWorkspaceConfigurationAsync(WorkspaceConfiguration configuration, ApiManifestDocument? manifestDocument, CancellationToken cancellationToken = default)
     {
         ArgumentNullException.ThrowIfNull(configuration);
-        if (!Directory.Exists(TargetDirectory))
-            Directory.CreateDirectory(TargetDirectory);
+        using (await localFilesLock.LockAsync(targetConfigurationFilePath, cancellationToken).ConfigureAwait(false))
+        {
+            if (!Directory.Exists(TargetDirectory))
+                Directory.CreateDirectory(TargetDirectory);
 #pragma warning disable CA2007
-        await using var configStream = File.Open(targetConfigurationFilePath, FileMode.Create);
+            await using var configStream = File.Open(targetConfigurationFilePath, FileMode.Create);
 #pragma warning restore CA2007
-        await JsonSerializer.SerializeAsync(configStream, configuration, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false);
-        if (manifestDocument != null)
-        {
+            await JsonSerializer.SerializeAsync(configStream, configuration, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false);
+            if (manifestDocument != null)
+            {
+                using (await localFilesLock.LockAsync(targetManifestFilePath, cancellationToken).ConfigureAwait(false))
+                {
 #pragma warning disable CA2007
-            await using var manifestStream = File.Open(targetManifestFilePath, FileMode.Create);
+                    await using var manifestStream = File.Open(targetManifestFilePath, FileMode.Create);
 #pragma warning restore CA2007
-            await manifestManagementService.SerializeManifestDocumentAsync(manifestDocument, manifestStream).ConfigureAwait(false);
+                    await manifestManagementService.SerializeManifestDocumentAsync(manifestDocument, manifestStream).ConfigureAwait(false);
+                }
+            }
         }
     }
     public Task<bool> IsInitializedAsync(CancellationToken cancellationToken = default)
@@ -64,51 +76,54 @@ public Task<bool> IsInitializedAsync(CancellationToken cancellationToken = defau
     public async Task<(WorkspaceConfiguration?, ApiManifestDocument?)> GetWorkspaceConfigurationAsync(CancellationToken cancellationToken = default)
     {
         if (File.Exists(targetConfigurationFilePath))
-        {
+            using (await localFilesLock.LockAsync(targetConfigurationFilePath, cancellationToken).ConfigureAwait(false))
+            {
 #pragma warning disable CA2007
-            await using var configStream = File.OpenRead(targetConfigurationFilePath);
+                await using var configStream = File.OpenRead(targetConfigurationFilePath);
 #pragma warning restore CA2007
-            var config = await JsonSerializer.DeserializeAsync(configStream, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false);
-            if (File.Exists(targetManifestFilePath))
-            {
+                var config = await JsonSerializer.DeserializeAsync(configStream, context.WorkspaceConfiguration, cancellationToken).ConfigureAwait(false);
+                if (File.Exists(targetManifestFilePath))
+                    using (await localFilesLock.LockAsync(targetManifestFilePath, cancellationToken).ConfigureAwait(false))
+                    {
 #pragma warning disable CA2007
-                await using var manifestStream = File.OpenRead(targetManifestFilePath);
+                        await using var manifestStream = File.OpenRead(targetManifestFilePath);
 #pragma warning restore CA2007
-                var manifest = await manifestManagementService.DeserializeManifestDocumentAsync(manifestStream).ConfigureAwait(false);
-                return (config, manifest);
+                        var manifest = await manifestManagementService.DeserializeManifestDocumentAsync(manifestStream).ConfigureAwait(false);
+                        return (config, manifest);
+                    }
+                return (config, null);
             }
-            return (config, null);
-        }
         return (null, null);
     }
-    public Task BackupConfigAsync(string directoryPath, CancellationToken cancellationToken = default)
+    public async Task BackupConfigAsync(string directoryPath, CancellationToken cancellationToken = default)
     {
         ArgumentException.ThrowIfNullOrEmpty(directoryPath);
-        BackupFile(directoryPath, ConfigurationFileName);
-        BackupFile(directoryPath, ManifestFileName);
-        return Task.CompletedTask;
+        await BackupFile(directoryPath, ConfigurationFileName, cancellationToken).ConfigureAwait(false);
+        await BackupFile(directoryPath, ManifestFileName, cancellationToken).ConfigureAwait(false);
     }
-    private static void BackupFile(string directoryPath, string fileName)
+    private static async Task BackupFile(string directoryPath, string fileName, CancellationToken cancellationToken = default)
     {
         var sourceFilePath = Path.Combine(directoryPath, fileName);
         if (File.Exists(sourceFilePath))
         {
             var backupFilePath = GetBackupFilePath(directoryPath, fileName);
-            var targetDirectory = Path.GetDirectoryName(backupFilePath);
-            if (string.IsNullOrEmpty(targetDirectory)) return;
-            if (!Directory.Exists(targetDirectory))
-                Directory.CreateDirectory(targetDirectory);
-            File.Copy(sourceFilePath, backupFilePath, true);
+            using (await localFilesLock.LockAsync(backupFilePath, cancellationToken).ConfigureAwait(false))
+            {
+                var targetDirectory = Path.GetDirectoryName(backupFilePath);
+                if (string.IsNullOrEmpty(targetDirectory)) return;
+                if (!Directory.Exists(targetDirectory))
+                    Directory.CreateDirectory(targetDirectory);
+                File.Copy(sourceFilePath, backupFilePath, true);
+            }
         }
     }
-    public Task RestoreConfigAsync(string directoryPath, CancellationToken cancellationToken = default)
+    public async Task RestoreConfigAsync(string directoryPath, CancellationToken cancellationToken = default)
     {
         ArgumentException.ThrowIfNullOrEmpty(directoryPath);
-        RestoreFile(directoryPath, ConfigurationFileName);
-        RestoreFile(directoryPath, ManifestFileName);
-        return Task.CompletedTask;
+        await RestoreFile(directoryPath, ConfigurationFileName, cancellationToken).ConfigureAwait(false);
+        await RestoreFile(directoryPath, ManifestFileName, cancellationToken).ConfigureAwait(false);
     }
-    private static void RestoreFile(string directoryPath, string fileName)
+    private static async Task RestoreFile(string directoryPath, string fileName, CancellationToken cancellationToken = default)
     {
         var sourceFilePath = Path.Combine(directoryPath, fileName);
         var targetDirectory = Path.GetDirectoryName(sourceFilePath);
@@ -118,7 +133,10 @@ private static void RestoreFile(string directoryPath, string fileName)
         var backupFilePath = GetBackupFilePath(directoryPath, fileName);
         if (File.Exists(backupFilePath))
         {
-            File.Copy(backupFilePath, sourceFilePath, true);
+            using (await localFilesLock.LockAsync(sourceFilePath, cancellationToken).ConfigureAwait(false))
+            {
+                File.Copy(backupFilePath, sourceFilePath, true);
+            }
         }
     }
     private static readonly ThreadLocal<HashAlgorithm> HashAlgorithm = new(SHA256.Create);