Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds support for OpenAI plugin generation #4656

Merged
merged 5 commits into from
May 15, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -352,7 +352,9 @@
"--type",
"ApiManifest",
"--type",
"microsoft"
"microsoft",
"--type",
"OpenAI"
],
"cwd": "${workspaceFolder}/samples/msgraph-mail/dotnet",
"console": "internalConsole",
Expand Down
2 changes: 1 addition & 1 deletion src/Kiota.Builder/Kiota.Builder.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
<PackageReference Include="Microsoft.OpenApi" Version="1.6.14" />
<PackageReference Include="Microsoft.OpenApi.ApiManifest" Version="0.5.5-preview" />
<PackageReference Include="Microsoft.OpenApi.Readers" Version="1.6.14" />
<PackageReference Include="Microsoft.Plugins.Manifest" Version="0.0.6-preview" />
<PackageReference Include="Microsoft.Plugins.Manifest" Version="0.0.7-preview" />
baywet marked this conversation as resolved.
Show resolved Hide resolved
<PackageReference Include="Microsoft.SourceLink.GitHub" Version="8.0.0" PrivateAssets="All" />
<PackageReference Include="YamlDotNet" Version="15.1.2" />
<ProjectReference Include="..\Kiota.Generated\KiotaGenerated.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
Expand Down
2 changes: 0 additions & 2 deletions src/Kiota.Builder/KiotaBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,6 @@
{
return await GenerateConsumerAsync(async (sw, stepId, openApiTree, CancellationToken) =>
{
if (config.PluginTypes.Contains(PluginType.OpenAI))
throw new NotImplementedException("The OpenAI plugin type is not supported for generation");
if (openApiDocument is null || openApiTree is null)
throw new InvalidOperationException("The OpenAPI document and the URL tree must be loaded before generating the plugins");
// generate plugin
Expand Down Expand Up @@ -659,7 +657,7 @@
methodToAdd.AddParameter(new CodeParameter
{
Name = "rawUrl",
Type = new CodeType { Name = "string", IsExternal = true },

Check warning on line 660 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Define a constant instead of using this literal 'string' 19 times. (https://rules.sonarsource.com/csharp/RSPEC-1192)
Optional = false,
Documentation = new()
{
Expand Down Expand Up @@ -1015,7 +1013,7 @@

if (!"string".Equals(parameter.Type.Name, StringComparison.OrdinalIgnoreCase) && config.IncludeBackwardCompatible)
{ // adding a second indexer for the string version of the parameter so we keep backward compatibility
//TODO remove for v2

Check warning on line 1016 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var backCompatibleValue = (CodeIndexer)result[0].Clone();
backCompatibleValue.Name += "-string";
backCompatibleValue.IndexParameter.Type = DefaultIndexerParameterType;
Expand Down Expand Up @@ -1169,7 +1167,7 @@
var suffix = $"{operationType}Response";
var modelType = CreateModelDeclarations(currentNode, schema, operation, parentClass, suffix);
if (modelType is not null && config.IncludeBackwardCompatible && config.Language is GenerationLanguage.CSharp or GenerationLanguage.Go && modelType.Name.EndsWith(suffix, StringComparison.Ordinal))
{ //TODO remove for v2

Check warning on line 1170 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var obsoleteTypeName = modelType.Name[..^suffix.Length] + "Response";
if (modelType is CodeType codeType &&
codeType.TypeDefinition is CodeClass codeClass)
Expand Down Expand Up @@ -1302,7 +1300,7 @@
executorMethod.AddParameter(cancellationParam);// Add cancellation token parameter

if (returnTypes.Item2 is not null && config.IncludeBackwardCompatible)
{ //TODO remove for v2

Check warning on line 1303 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var additionalExecutorMethod = (CodeMethod)executorMethod.Clone();
additionalExecutorMethod.ReturnType = returnTypes.Item2;
additionalExecutorMethod.OriginalMethod = executorMethod;
Expand Down Expand Up @@ -2340,7 +2338,7 @@
if (!parameterClass.ContainsPropertyWithWireName(prop.WireName))
{
if (addBackwardCompatibleParameter && config.IncludeBackwardCompatible && config.Language is GenerationLanguage.CSharp or GenerationLanguage.Go)
{ //TODO remove for v2

Check warning on line 2341 in src/Kiota.Builder/KiotaBuilder.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
var modernProp = (CodeProperty)prop.Clone();
modernProp.Name = $"{prop.Name}As{modernProp.Type.Name.ToFirstCharacterUpperCase()}";
modernProp.SerializationName = prop.WireName;
Expand Down
105 changes: 78 additions & 27 deletions src/Kiota.Builder/Plugins/PluginsGenerationService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
private static readonly OpenAPIRuntimeComparer _openAPIRuntimeComparer = new();
private const string ManifestFileNameSuffix = ".json";
private const string DescriptionPathSuffix = "openapi.yml";
private const string OpenAIManifestFileName = "openai-plugins";
public async Task GenerateManifestAsync(CancellationToken cancellationToken = default)
{
// write the description
Expand All @@ -56,7 +57,8 @@
// write the plugins
foreach (var pluginType in Configuration.PluginTypes)
{
var manifestOutputPath = Path.Combine(Configuration.OutputPath, $"{Configuration.ClientClassName.ToLowerInvariant()}-{pluginType.ToString().ToLowerInvariant()}{ManifestFileNameSuffix}");
var manifestFileName = pluginType == PluginType.OpenAI ? OpenAIManifestFileName : $"{Configuration.ClientClassName.ToLowerInvariant()}-{pluginType.ToString().ToLowerInvariant()}";
var manifestOutputPath = Path.Combine(Configuration.OutputPath, $"{manifestFileName}{ManifestFileNameSuffix}");
#pragma warning disable CA2007 // Consider calling ConfigureAwait on the awaited task
await using var fileStream = File.Create(manifestOutputPath, 4096);
await using var writer = new Utf8JsonWriter(fileStream, new JsonWriterOptions { Indented = true });
Expand All @@ -69,52 +71,68 @@
pluginDocument.Write(writer);
break;
case PluginType.APIManifest:
var apiManifest = new ApiManifestDocument("application"); //TODO add application name

Check warning on line 74 in src/Kiota.Builder/Plugins/PluginsGenerationService.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
// pass empty cong hash so that its not included in this manifest.
// pass empty config hash so that its not included in this manifest.
apiManifest.ApiDependencies.AddOrReplace(Configuration.ClientClassName, Configuration.ToApiDependency(string.Empty, TreeNode?.GetRequestInfo().ToDictionary(static x => x.Key, static x => x.Value) ?? [], WorkingDirectory));
var publisherName = string.IsNullOrEmpty(OAIDocument.Info?.Contact?.Name)
? DefaultContactName
: OAIDocument.Info.Contact.Name;
var publisherEmail = string.IsNullOrEmpty(OAIDocument.Info?.Contact?.Email)
? DefaultContactEmail
: OAIDocument.Info.Contact.Email;
apiManifest.Publisher = new Publisher(publisherName, publisherEmail);
apiManifest.Write(writer);
break;
case PluginType.OpenAI://TODO add support for OpenAI plugin type generation
// intentional drop to the default case
case PluginType.OpenAI:
var pluginDocumentV1 = GetV1ManifestDocument(descriptionRelativePath);
pluginDocumentV1.Write(writer);
break;
default:
throw new NotImplementedException($"The {pluginType} plugin is not implemented.");
}
await writer.FlushAsync(cancellationToken).ConfigureAwait(false);
}
}
private PluginManifestDocument GetV1ManifestDocument(string openApiDocumentPath)
{
var descriptionForHuman = OAIDocument.Info?.Description.CleanupXMLString() is string d && !string.IsNullOrEmpty(d) ? d : $"Description for {OAIDocument.Info?.Title.CleanupXMLString()}";
var manifestInfo = ExtractInfoFromDocument(OAIDocument.Info);
return new PluginManifestDocument
{
SchemaVersion = "v1",
NameForHuman = OAIDocument.Info?.Title.CleanupXMLString(),
NameForModel = OAIDocument.Info?.Title.CleanupXMLString(),
DescriptionForHuman = descriptionForHuman,
DescriptionForModel = manifestInfo.DescriptionForModel ?? descriptionForHuman,
Auth = new V1AnonymousAuth(),
Api = new Api()
{
Type = ApiType.openapi,
URL = openApiDocumentPath
},
ContactEmail = manifestInfo.ContactEmail,
LogoUrl = manifestInfo.LogoUrl,
LegalInfoUrl = manifestInfo.LegalUrl,
};
}

private PluginManifestDocument GetManifestDocument(string openApiDocumentPath)
{
var (runtimes, functions) = GetRuntimesAndFunctionsFromTree(TreeNode, openApiDocumentPath);
var descriptionForHuman = OAIDocument.Info?.Description.CleanupXMLString() is string d && !string.IsNullOrEmpty(d) ? d : $"Description for {OAIDocument.Info?.Title.CleanupXMLString()}";
var descriptionForModel = descriptionForHuman;
string? legalUrl = null;
string? logoUrl = null;
string? privacyUrl = null;
if (OAIDocument.Info is not null)
{
if (OAIDocument.Info.Extensions.TryGetValue(OpenApiDescriptionForModelExtension.Name, out var descriptionExtension) &&
descriptionExtension is OpenApiDescriptionForModelExtension extension &&
!string.IsNullOrEmpty(extension.Description))
descriptionForModel = extension.Description.CleanupXMLString();
if (OAIDocument.Info.Extensions.TryGetValue(OpenApiLegalInfoUrlExtension.Name, out var legalExtension) && legalExtension is OpenApiLegalInfoUrlExtension legal)
legalUrl = legal.Legal;
if (OAIDocument.Info.Extensions.TryGetValue(OpenApiLogoExtension.Name, out var logoExtension) && logoExtension is OpenApiLogoExtension logo)
logoUrl = logo.Url;
if (OAIDocument.Info.Extensions.TryGetValue(OpenApiPrivacyPolicyUrlExtension.Name, out var privacyExtension) && privacyExtension is OpenApiPrivacyPolicyUrlExtension privacy)
privacyUrl = privacy.Privacy;
}
var manifestInfo = ExtractInfoFromDocument(OAIDocument.Info);
return new PluginManifestDocument
{
SchemaVersion = "v2",
NameForHuman = OAIDocument.Info?.Title.CleanupXMLString(),
// TODO name for model ???

Check warning on line 128 in src/Kiota.Builder/Plugins/PluginsGenerationService.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
DescriptionForHuman = descriptionForHuman,
DescriptionForModel = descriptionForModel,
ContactEmail = OAIDocument.Info?.Contact?.Email,
DescriptionForModel = manifestInfo.DescriptionForModel ?? descriptionForHuman,
ContactEmail = manifestInfo.ContactEmail,
Namespace = Configuration.ClientClassName,
LogoUrl = logoUrl,
LegalInfoUrl = legalUrl,
PrivacyPolicyUrl = privacyUrl,
LogoUrl = manifestInfo.LogoUrl,
LegalInfoUrl = manifestInfo.LegalUrl,
PrivacyPolicyUrl = manifestInfo.PrivacyUrl,
Runtimes = [.. runtimes
.GroupBy(static x => x, _openAPIRuntimeComparer)
.Select(static x =>
Expand All @@ -127,7 +145,40 @@
Functions = [.. functions.OrderBy(static x => x.Name, StringComparer.OrdinalIgnoreCase)]
};
}
private (OpenApiRuntime[], Function[]) GetRuntimesAndFunctionsFromTree(OpenApiUrlTreeNode currentNode, string openApiDocumentPath)

private static OpenApiManifestInfo ExtractInfoFromDocument(OpenApiInfo? openApiInfo)
{
var manifestInfo = new OpenApiManifestInfo();

if (openApiInfo is null)
return manifestInfo;

string? descriptionForModel = null;
string? legalUrl = null;
string? logoUrl = null;
string? privacyUrl = null;
string contactEmail = string.IsNullOrEmpty(openApiInfo.Contact?.Email)
? DefaultContactEmail
: openApiInfo.Contact.Email;

if (openApiInfo.Extensions.TryGetValue(OpenApiDescriptionForModelExtension.Name, out var descriptionExtension) &&
descriptionExtension is OpenApiDescriptionForModelExtension extension &&
!string.IsNullOrEmpty(extension.Description))
descriptionForModel = extension.Description.CleanupXMLString();
if (openApiInfo.Extensions.TryGetValue(OpenApiLegalInfoUrlExtension.Name, out var legalExtension) && legalExtension is OpenApiLegalInfoUrlExtension legal)
legalUrl = legal.Legal;
if (openApiInfo.Extensions.TryGetValue(OpenApiLogoExtension.Name, out var logoExtension) && logoExtension is OpenApiLogoExtension logo)
logoUrl = logo.Url;
if (openApiInfo.Extensions.TryGetValue(OpenApiPrivacyPolicyUrlExtension.Name, out var privacyExtension) && privacyExtension is OpenApiPrivacyPolicyUrlExtension privacy)
privacyUrl = privacy.Privacy;

return new OpenApiManifestInfo(descriptionForModel, legalUrl, logoUrl, privacyUrl, contactEmail);

}
private const string DefaultContactName = "publisher-name";
private const string DefaultContactEmail = "[email protected]";
private sealed record OpenApiManifestInfo(string? DescriptionForModel = null, string? LegalUrl = null, string? LogoUrl = null, string? PrivacyUrl = null, string ContactEmail = DefaultContactEmail);
private static (OpenApiRuntime[], Function[]) GetRuntimesAndFunctionsFromTree(OpenApiUrlTreeNode currentNode, string openApiDocumentPath)
{
var runtimes = new List<OpenApiRuntime>();
var functions = new List<Function>();
Expand All @@ -148,7 +199,7 @@
.Union(pathItem.Parameters.Where(static x => x.In is ParameterLocation.Path))
.Where(static x => x.Schema?.Type is not null && scalarTypes.Contains(x.Schema.Type))
.ToArray();
//TODO add request body

Check warning on line 202 in src/Kiota.Builder/Plugins/PluginsGenerationService.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)

functions.Add(new Function
{
Expand All @@ -169,7 +220,7 @@
Type = x.Schema.Type ?? string.Empty,
Description = x.Description.CleanupXMLString(),
Default = x.Schema.Default?.ToString() ?? string.Empty,
//TODO enums

Check warning on line 223 in src/Kiota.Builder/Plugins/PluginsGenerationService.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
})),
Required = oasParameters.Where(static x => x.Required).Select(static x => x.Name).ToList()
},
Expand Down Expand Up @@ -220,5 +271,5 @@
return null;
}
private static readonly HashSet<string> scalarTypes = new(StringComparer.OrdinalIgnoreCase) { "string", "number", "integer", "boolean" };
//TODO validate this is right, in OAS integer are under type number for the json schema, but integer is ok for query parameters

Check warning on line 274 in src/Kiota.Builder/Plugins/PluginsGenerationService.cs

View workflow job for this annotation

GitHub Actions / Build

Complete the task associated to this 'TODO' comment. (https://rules.sonarsource.com/csharp/RSPEC-1135)
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,23 @@ public async Task GeneratesManifest()
paths:
/test:
get:
description: description for test path
operationId: test
responses:
'200':
description: test
/test/{id}:
get:
description: description for test path with id
operationId: test_WithId
parameters:
- name: id
in: path
required: true
description: The id of the test
schema:
type: integer
format: int32
responses:
'200':
description: test";
Expand All @@ -57,7 +73,7 @@ public async Task GeneratesManifest()
{
OutputPath = outputDirectory,
OpenAPIFilePath = "openapiPath",
PluginTypes = [PluginType.Microsoft, PluginType.APIManifest],
PluginTypes = [PluginType.Microsoft, PluginType.APIManifest, PluginType.OpenAI],
ClientClassName = "client",
ApiRootUrl = "http://localhost/", //Kiota builder would set this for us
};
Expand All @@ -70,12 +86,26 @@ public async Task GeneratesManifest()

Assert.True(File.Exists(Path.Combine(outputDirectory, ManifestFileName)));
Assert.True(File.Exists(Path.Combine(outputDirectory, "client-apimanifest.json")));
Assert.True(File.Exists(Path.Combine(outputDirectory, OpenAIPluginFileName)));
Assert.True(File.Exists(Path.Combine(outputDirectory, OpenApiFileName)));

// Validate the v2 plugin
var manifestContent = await File.ReadAllTextAsync(Path.Combine(outputDirectory, ManifestFileName));
using var jsonDocument = JsonDocument.Parse(manifestContent);
var resultingManifest = PluginManifestDocument.Load(jsonDocument.RootElement);
Assert.NotNull(resultingManifest.Document);
Assert.Equal(OpenApiFileName, resultingManifest.Document.Runtimes.OfType<OpenApiRuntime>().First().Spec.Url);
Assert.Empty(resultingManifest.Problems);

// Validate the v1 plugin
var v1ManifestContent = await File.ReadAllTextAsync(Path.Combine(outputDirectory, OpenAIPluginFileName));
using var v1JsonDocument = JsonDocument.Parse(v1ManifestContent);
var v1Manifest = PluginManifestDocument.Load(v1JsonDocument.RootElement);
Assert.NotNull(resultingManifest.Document);
Assert.Equal(OpenApiFileName, v1Manifest.Document.Api.URL);
Assert.Empty(v1Manifest.Problems);
}
private const string ManifestFileName = "client-microsoft.json";
private const string OpenAIPluginFileName = "openai-plugins.json";
private const string OpenApiFileName = "client-openapi.yml";
}
Loading