Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature/plugins #4416

Merged
merged 50 commits into from
Apr 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
908fb2e
- adds basic command infrastructure for plugins
baywet Mar 25, 2024
b1a484b
- code linting
baywet Mar 25, 2024
33e780f
- maps plugin remove command
baywet Mar 25, 2024
b020268
- implements the plugin generate command
baywet Mar 25, 2024
9aa7482
- adds launch configuration for plugin add
baywet Mar 25, 2024
3300d15
- implements plugin add command handler body
baywet Mar 25, 2024
bba96d6
- fixes should generate implementation for plugins
baywet Mar 25, 2024
967ed84
- fixes formatting
baywet Mar 26, 2024
66e7d07
- initial plugin generation
baywet Mar 26, 2024
e1c17d2
- adds parameters to the plugin
baywet Mar 26, 2024
bc2dce7
- adds support for descriptive properties
baywet Mar 26, 2024
84ad261
- dedupes runtimes for plugin generation
baywet Mar 27, 2024
8ce7f74
- code linting
baywet Mar 27, 2024
59817d5
- adds AI extensions support
baywet Mar 27, 2024
29fad06
- overrides description for model by equivalent extension when present
baywet Mar 27, 2024
7910deb
- adds support for instructions
baywet Mar 27, 2024
f661f68
- adds copy of sliced description
baywet Mar 27, 2024
5e4c4d0
- updates output directory for descriptions copies
baywet Mar 27, 2024
f63190c
- replaces project reference by package
baywet Mar 28, 2024
39c092a
- adds validation error for OpenAI plugins
baywet Mar 28, 2024
74d62ea
- implements plugin edit command
baywet Mar 28, 2024
d564e4a
- adds missing unit tests for serialization of extensions
baywet Mar 28, 2024
61da724
- adds unit tests for plugin removal
baywet Mar 28, 2024
3aa5ef1
- code linting
baywet Mar 28, 2024
1c06fd9
- adds unit tests for plugin configuration
baywet Mar 28, 2024
6a71f06
- adds unit tests for plugin comparer
baywet Mar 28, 2024
8bcc32f
- adds unit tests for runtime comparer
baywet Mar 28, 2024
2f924ed
- adds unit tests for plugin generation service
baywet Mar 28, 2024
9193228
- adds missing operation id
baywet Mar 28, 2024
643f9ee
- adds support for logo, privacy, legal urls
baywet Mar 28, 2024
0be5ce6
Merge branch 'main' into feature/plugins
andrueastman Apr 9, 2024
e23d1ce
Merge remote-tracking branch 'origin/main' into feature/plugins
Apr 11, 2024
cab61c9
No defaults for plugin types
Apr 11, 2024
35e1b49
Update logo extension
Apr 11, 2024
68a3557
Fixes breaking changes from latest update from manifest lib
Apr 16, 2024
9f9d0bb
Fix test
Apr 16, 2024
6380cef
Merge pull request #4484 from microsoft/andrueastman/pluginFixes
baywet Apr 16, 2024
0273224
Merge branch 'feature/plugins' into andrueastman/manifestLibUpdates
andrueastman Apr 17, 2024
0fe8875
Bump manifest and plugins versions
Apr 17, 2024
bbf9f45
Merge branch 'main' into feature/plugins
andrueastman Apr 18, 2024
91325a6
Merge branch 'feature/plugins' into andrueastman/manifestLibUpdates
andrueastman Apr 18, 2024
691a588
Add support for api manifest generation
Apr 18, 2024
a065bde
Adds support for manifest generation
Apr 18, 2024
fbb3fd8
Merge branch 'main' into feature/plugins
andrueastman Apr 19, 2024
b59fce3
Merge branch 'feature/plugins' into andrueastman/manifestLibUpdates
andrueastman Apr 19, 2024
6cbc453
Throw error for not supported case
Apr 19, 2024
3c5852a
Fix formatting
Apr 19, 2024
b333824
Merge pull request #4499 from microsoft/andrueastman/manifestLibUpdates
baywet Apr 19, 2024
c9aa347
Update src/Kiota.Builder/WorkspaceManagement/DescriptionStorageServic…
baywet Apr 19, 2024
af6f755
Merge branch 'main' into feature/plugins
baywet Apr 19, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 59 additions & 8 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,10 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["search", "microsoft"],
"args": [
"search",
"microsoft"
],
"cwd": "${workspaceFolder}/src/kiota",
"console": "internalConsole",
"stopAtEntry": false
Expand All @@ -207,7 +210,10 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["search", "test"],
"args": [
"search",
"test"
],
"cwd": "${workspaceFolder}/src/kiota",
"console": "internalConsole",
"stopAtEntry": false
Expand Down Expand Up @@ -249,7 +255,11 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["info", "-l", "CSharp"],
"args": [
"info",
"-l",
"CSharp"
],
"cwd": "${workspaceFolder}/src/kiota",
"console": "internalConsole",
"stopAtEntry": false
Expand All @@ -260,7 +270,11 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["update", "-o", "${workspaceFolder}/samples"],
"args": [
"update",
"-o",
"${workspaceFolder}/samples"
],
"cwd": "${workspaceFolder}/src/kiota",
"console": "internalConsole",
"stopAtEntry": false
Expand All @@ -271,7 +285,10 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["workspace", "migrate"],
"args": [
"workspace",
"migrate"
],
"cwd": "${workspaceFolder}/samples/msgraph-mail/dotnet",
"console": "internalConsole",
"stopAtEntry": false,
Expand All @@ -285,7 +302,10 @@
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["client", "generate"],
"args": [
"client",
"generate"
],
"cwd": "${workspaceFolder}/samples/msgraph-mail/dotnet",
"console": "internalConsole",
"stopAtEntry": false,
Expand Down Expand Up @@ -314,13 +334,44 @@
"KIOTA_CONFIG_PREVIEW": "true"
}
},
{
"name": "Launch Plugin Add",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": [
"plugin",
"add",
"--plugin-name",
"MicrosoftGraph",
"-d",
"https://raw.githubusercontent.com/microsoftgraph/msgraph-sdk-powershell/dev/openApiDocs/v1.0/Mail.yml",
"-i",
"**/messages",
"--type",
"ApiManifest",
"--type",
"microsoft"
],
"cwd": "${workspaceFolder}/samples/msgraph-mail/dotnet",
"console": "internalConsole",
"stopAtEntry": false,
"env": {
"KIOTA_CONFIG_PREVIEW": "true"
}
},
{
"name": "Launch Login (github - device)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
"program": "${workspaceFolder}/src/kiota/bin/Debug/net8.0/kiota.dll",
"args": ["login", "github", "device"],
"args": [
"login",
"github",
"device"
],
"cwd": "${workspaceFolder}/src/kiota",
"console": "internalConsole",
"stopAtEntry": false
Expand All @@ -332,4 +383,4 @@
"processId": "${command:pickProcess}"
}
]
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
namespace Kiota.Builder.Configuration;
public enum ClientOperation
public enum ConsumerOperation
{
Add,
Edit,
Expand Down
7 changes: 4 additions & 3 deletions src/Kiota.Builder/Configuration/GenerationConfiguration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

namespace Kiota.Builder.Configuration;
#pragma warning disable CA2227
#pragma warning disable CA1002
#pragma warning disable CA1056
public class GenerationConfiguration : ICloneable
{
Expand All @@ -30,7 +29,7 @@ public bool SkipGeneration
{
get; set;
}
public ClientOperation? Operation
public ConsumerOperation? Operation
{
get; set;
}
Expand All @@ -46,6 +45,7 @@ public string ModelsNamespaceName
get => $"{ClientNamespaceName}{NamespaceNameSeparator}{ModelsNamespaceSegmentName}";
}
public GenerationLanguage Language { get; set; } = GenerationLanguage.CSharp;
public HashSet<PluginType> PluginTypes { get; set; } = [];
public string? ApiRootUrl
{
get; set;
Expand Down Expand Up @@ -150,6 +150,7 @@ public object Clone()
SkipGeneration = SkipGeneration,
Operation = Operation,
PatternsOverride = new(PatternsOverride ?? Enumerable.Empty<string>(), StringComparer.OrdinalIgnoreCase),
PluginTypes = new(PluginTypes ?? Enumerable.Empty<PluginType>()),
};
}
private static readonly StringIEnumerableDeepComparer comparer = new();
Expand Down Expand Up @@ -185,7 +186,7 @@ public ApiDependency ToApiDependency(string configurationHash, Dictionary<string
};
return dependency;
}
public bool IsPluginConfiguration => PluginTypes.Count != 0;
}
#pragma warning restore CA1056
#pragma warning restore CA1002
#pragma warning restore CA2227
6 changes: 4 additions & 2 deletions src/Kiota.Builder/Kiota.Builder.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -47,14 +47,16 @@
<PackageReference Include="Microsoft.OpenApi" Version="1.6.14" />
<PackageReference Include="Microsoft.OpenApi.ApiManifest" Version="0.5.4-preview" />
<PackageReference Include="Microsoft.OpenApi.Readers" Version="1.6.14" />
<PackageReference Include="Microsoft.Plugins.Manifest" Version="0.0.6-preview" />
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
<PackageReference Include="YamlDotNet" Version="15.1.2" />
<ProjectReference Include="..\Kiota.Generated\KiotaGenerated.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<ProjectReference Include="..\Kiota.Generated\KiotaGenerated.csproj" OutputItemType="Analyzer"
ReferenceOutputAssembly="false" />
</ItemGroup>
<ItemGroup>
<AdditionalFiles Include="*.g.cs" />
</ItemGroup>
<ItemGroup>
<None Include="../../README.md" Pack="true" PackagePath="" />
</ItemGroup>
</Project>
</Project>
62 changes: 46 additions & 16 deletions src/Kiota.Builder/KiotaBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
using Kiota.Builder.Logging;
using Kiota.Builder.Manifest;
using Kiota.Builder.OpenApiExtensions;
using Kiota.Builder.Plugins;
using Kiota.Builder.Refiners;
using Kiota.Builder.WorkspaceManagement;
using Kiota.Builder.Writers;
Expand Down Expand Up @@ -221,19 +222,61 @@
return kiotaExt.LanguagesInformation;
return null;
}
/// <summary>
/// Generates the API plugins from the OpenAPI document
/// </summary>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>Whether the generated plugin was updated or not</returns>
public async Task<bool> GeneratePluginAsync(CancellationToken cancellationToken)
{
return await GenerateConsumerAsync(async (sw, stepId, openApiTree, CancellationToken) =>
{
if (config.PluginTypes.Contains(PluginType.OpenAI))
throw new NotImplementedException("The OpenAI plugin type is not supported for generation");
if (openApiDocument is null || openApiTree is null)
throw new InvalidOperationException("The OpenAPI document and the URL tree must be loaded before generating the plugins");
// generate plugin
sw.Start();
var pluginsService = new PluginsGenerationService(openApiDocument, openApiTree, config);
await pluginsService.GenerateManifestAsync(cancellationToken).ConfigureAwait(false);
StopLogAndReset(sw, $"step {++stepId} - generate plugin - took");
return stepId;
}, cancellationToken).ConfigureAwait(false);
}

/// <summary>
/// Generates the code from the OpenAPI document
/// </summary>
/// <param name="cancellationToken">The cancellation token</param>
/// <returns>Whether the generated code was updated or not</returns>
public async Task<bool> GenerateClientAsync(CancellationToken cancellationToken)
{
return await GenerateConsumerAsync(async (sw, stepId, openApiTree, CancellationToken) =>
{
// Create Source Model
sw.Start();
var generatedCode = CreateSourceModel(openApiTree);
StopLogAndReset(sw, $"step {++stepId} - create source model - took");

// RefineByLanguage
sw.Start();
await ApplyLanguageRefinement(config, generatedCode, cancellationToken).ConfigureAwait(false);
StopLogAndReset(sw, $"step {++stepId} - refine by language - took");

// Write language source
sw.Start();
await CreateLanguageSourceFilesAsync(config.Language, generatedCode, cancellationToken).ConfigureAwait(false);
StopLogAndReset(sw, $"step {++stepId} - writing files - took");
return stepId;
}, cancellationToken).ConfigureAwait(false);
}
private async Task<bool> GenerateConsumerAsync(Func<Stopwatch, int, OpenApiUrlTreeNode?, CancellationToken, Task<int>> innerGenerationSteps, CancellationToken cancellationToken)
{
var sw = new Stopwatch();
// Read input stream
var inputPath = config.OpenAPIFilePath;

if (config.Operation is ClientOperation.Add && await workspaceManagementService.IsClientPresent(config.ClientClassName, cancellationToken).ConfigureAwait(false))
if (config.Operation is ConsumerOperation.Add && await workspaceManagementService.IsConsumerPresent(config.ClientClassName, cancellationToken).ConfigureAwait(false))
throw new InvalidOperationException($"The client {config.ClientClassName} already exists in the workspace");

try
Expand All @@ -252,27 +295,14 @@

if (shouldGenerate)
{
// Create Source Model
sw.Start();
var generatedCode = CreateSourceModel(openApiTree);
StopLogAndReset(sw, $"step {++stepId} - create source model - took");

// RefineByLanguage
sw.Start();
await ApplyLanguageRefinement(config, generatedCode, cancellationToken).ConfigureAwait(false);
StopLogAndReset(sw, $"step {++stepId} - refine by language - took");

// Write language source
sw.Start();
await CreateLanguageSourceFilesAsync(config.Language, generatedCode, cancellationToken).ConfigureAwait(false);
StopLogAndReset(sw, $"step {++stepId} - writing files - took");
stepId = await innerGenerationSteps(sw, stepId, openApiTree, cancellationToken).ConfigureAwait(false);

await FinalizeWorkspaceAsync(sw, stepId, openApiTree, inputPath, cancellationToken).ConfigureAwait(false);
}
else
{
logger.LogInformation("No changes detected, skipping generation");
if (config.Operation is ClientOperation.Add or ClientOperation.Edit && config.SkipGeneration)
if (config.Operation is ConsumerOperation.Add or ConsumerOperation.Edit && config.SkipGeneration)
{
await FinalizeWorkspaceAsync(sw, stepId, openApiTree, inputPath, cancellationToken).ConfigureAwait(false);
}
Expand Down Expand Up @@ -976,7 +1006,7 @@

if (!"string".Equals(parameter.Type.Name, StringComparison.OrdinalIgnoreCase) && config.IncludeBackwardCompatible)
{ // adding a second indexer for the string version of the parameter so we keep backward compatibility
//TODO remove for v2

Check warning on line 1009 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var backCompatibleValue = (CodeIndexer)result[0].Clone();
backCompatibleValue.Name += "-string";
backCompatibleValue.IndexParameter.Type = DefaultIndexerParameterType;
Expand Down Expand Up @@ -1130,7 +1160,7 @@
var suffix = $"{operationType}Response";
var modelType = CreateModelDeclarations(currentNode, schema, operation, parentClass, suffix);
if (modelType is not null && config.IncludeBackwardCompatible && config.Language is GenerationLanguage.CSharp or GenerationLanguage.Go && modelType.Name.EndsWith(suffix, StringComparison.Ordinal))
{ //TODO remove for v2

Check warning on line 1163 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var obsoleteTypeName = modelType.Name[..^suffix.Length] + "Response";
if (modelType is CodeType codeType &&
codeType.TypeDefinition is CodeClass codeClass)
Expand Down Expand Up @@ -1263,7 +1293,7 @@
executorMethod.AddParameter(cancellationParam);// Add cancellation token parameter

if (returnTypes.Item2 is not null && config.IncludeBackwardCompatible)
{ //TODO remove for v2

Check warning on line 1296 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var additionalExecutorMethod = (CodeMethod)executorMethod.Clone();
additionalExecutorMethod.ReturnType = returnTypes.Item2;
additionalExecutorMethod.OriginalMethod = executorMethod;
Expand Down Expand Up @@ -2272,7 +2302,7 @@
if (!parameterClass.ContainsPropertyWithWireName(prop.WireName))
{
if (addBackwardCompatibleParameter && config.IncludeBackwardCompatible && config.Language is GenerationLanguage.CSharp or GenerationLanguage.Go)
{ //TODO remove for v2

Check warning on line 2305 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var modernProp = (CodeProperty)prop.Clone();
modernProp.Name = $"{prop.Name}As{modernProp.Type.Name.ToFirstCharacterUpperCase()}";
modernProp.SerializationName = prop.WireName;
Expand Down
8 changes: 7 additions & 1 deletion src/Kiota.Builder/OpenApiDocumentDownloadService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public OpenApiDocumentDownloadService(HttpClient httpClient, ILogger logger)
Stream input;
var isDescriptionFromWorkspaceCopy = false;
if (useKiotaConfig &&
config.Operation is ClientOperation.Edit or ClientOperation.Add &&
config.Operation is ConsumerOperation.Edit or ConsumerOperation.Add &&
workspaceManagementService is not null &&
await workspaceManagementService.GetDescriptionCopyAsync(config.ClientClassName, inputPath, config.CleanOutput, cancellationToken).ConfigureAwait(false) is { } descriptionStream)
{
Expand Down Expand Up @@ -114,6 +114,12 @@ ex is SecurityException ||
};
settings.AddMicrosoftExtensionParsers();
settings.ExtensionParsers.TryAdd(OpenApiKiotaExtension.Name, static (i, _) => OpenApiKiotaExtension.Parse(i));
settings.ExtensionParsers.TryAdd(OpenApiDescriptionForModelExtension.Name, static (i, _) => OpenApiDescriptionForModelExtension.Parse(i));
settings.ExtensionParsers.TryAdd(OpenApiLogoExtension.Name, static (i, _) => OpenApiLogoExtension.Parse(i));
settings.ExtensionParsers.TryAdd(OpenApiPrivacyPolicyUrlExtension.Name, static (i, _) => OpenApiPrivacyPolicyUrlExtension.Parse(i));
settings.ExtensionParsers.TryAdd(OpenApiLegalInfoUrlExtension.Name, static (i, _) => OpenApiLegalInfoUrlExtension.Parse(i));
settings.ExtensionParsers.TryAdd(OpenApiAiReasoningInstructionsExtension.Name, static (i, _) => OpenApiAiReasoningInstructionsExtension.Parse(i));
settings.ExtensionParsers.TryAdd(OpenApiAiRespondingInstructionsExtension.Name, static (i, _) => OpenApiAiRespondingInstructionsExtension.Parse(i));
try
{
var rawUri = config.OpenAPIFilePath.TrimEnd(KiotaBuilder.ForwardSlash);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.OpenApi;
using Microsoft.OpenApi.Any;
using Microsoft.OpenApi.Interfaces;
using Microsoft.OpenApi.Writers;

namespace Kiota.Builder.OpenApiExtensions;

public class OpenApiAiReasoningInstructionsExtension : IOpenApiExtension
{
public static string Name => "x-ai-reasoning-instructions";
#pragma warning disable CA1002 // Do not expose generic lists
public List<string> ReasoningInstructions { get; init; } = [];
#pragma warning restore CA1002 // Do not expose generic lists
public void Write(IOpenApiWriter writer, OpenApiSpecVersion specVersion)
{
ArgumentNullException.ThrowIfNull(writer);
if (ReasoningInstructions != null &&
ReasoningInstructions.Count != 0)
{
writer.WriteStartArray();
foreach (var instruction in ReasoningInstructions)
{
writer.WriteValue(instruction);
}
writer.WriteEndArray();
}
}
public static OpenApiAiReasoningInstructionsExtension Parse(IOpenApiAny source)
{
if (source is not OpenApiArray rawArray) throw new ArgumentOutOfRangeException(nameof(source));
var result = new OpenApiAiReasoningInstructionsExtension();
result.ReasoningInstructions.AddRange(rawArray.OfType<OpenApiString>().Select(x => x.Value));
return result;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.OpenApi;
using Microsoft.OpenApi.Any;
using Microsoft.OpenApi.Interfaces;
using Microsoft.OpenApi.Writers;

namespace Kiota.Builder.OpenApiExtensions;

public class OpenApiAiRespondingInstructionsExtension : IOpenApiExtension
{
public static string Name => "x-ai-responding-instructions";
#pragma warning disable CA1002 // Do not expose generic lists
public List<string> RespondingInstructions { get; init; } = [];
#pragma warning restore CA1002 // Do not expose generic lists
public void Write(IOpenApiWriter writer, OpenApiSpecVersion specVersion)
{
ArgumentNullException.ThrowIfNull(writer);
if (RespondingInstructions != null &&
RespondingInstructions.Count != 0)
{
writer.WriteStartArray();
foreach (var instruction in RespondingInstructions)
{
writer.WriteValue(instruction);
}
writer.WriteEndArray();
}
}
public static OpenApiAiRespondingInstructionsExtension Parse(IOpenApiAny source)
{
if (source is not OpenApiArray rawArray) throw new ArgumentOutOfRangeException(nameof(source));
var result = new OpenApiAiRespondingInstructionsExtension();
result.RespondingInstructions.AddRange(rawArray.OfType<OpenApiString>().Select(x => x.Value));
return result;
}
}
Loading
Loading