From c09602178f609146c0532d41b437a38d38053c05 Mon Sep 17 00:00:00 2001 From: Jeremy LaCivita Date: Tue, 25 Jun 2024 12:53:34 -0400 Subject: [PATCH] feat: Merge anyOf and oneOfs that are mergeable if enableUniionTypes is false --- languages/cpp/language.config.json | 3 +- languages/javascript/language.config.json | 1 + src/macrofier/engine.mjs | 76 ++-- src/macrofier/index.mjs | 21 +- src/macrofier/types.mjs | 64 ++- src/openrpc/index.mjs | 2 +- src/sdk/index.mjs | 4 +- src/shared/json-schema.mjs | 527 ++++++++++++++++++---- src/shared/modules.mjs | 175 +------ 9 files changed, 563 insertions(+), 310 deletions(-) diff --git a/languages/cpp/language.config.json b/languages/cpp/language.config.json index 724a1359..0f7cee05 100644 --- a/languages/cpp/language.config.json +++ b/languages/cpp/language.config.json @@ -4,9 +4,8 @@ "createModuleDirectories": false, "extractSubSchemas": true, "unwrapResultObjects": false, - "createPolymorphicMethods": true, + "enableUnionTypes": false, "excludeDeclarations": true, - "extractProviderSchema": true, "aggregateFiles": [ "/include/firebolt.h", "/src/firebolt.cpp" diff --git a/languages/javascript/language.config.json b/languages/javascript/language.config.json index 12bd94e8..360f0ec9 100644 --- a/languages/javascript/language.config.json +++ b/languages/javascript/language.config.json @@ -10,6 +10,7 @@ ], "createModuleDirectories": true, "copySchemasIntoModules": false, + "enableUnionTypes": true, "mergeOnTitle": true, "aggregateFiles": [ "/index.d.ts" diff --git a/src/macrofier/engine.mjs b/src/macrofier/engine.mjs index 8f71a5a8..8029ee84 100644 --- a/src/macrofier/engine.mjs +++ b/src/macrofier/engine.mjs @@ -29,10 +29,10 @@ import isString from 'crocks/core/isString.js' import predicates from 'crocks/predicates/index.js' const { isObject, isArray, propEq, pathSatisfies, propSatisfies } = predicates -import { isRPCOnlyMethod, isProviderInterfaceMethod, getProviderInterface, getPayloadFromEvent, providerHasNoParameters, isTemporalSetMethod, hasMethodAttributes, getMethodAttributes, isEventMethodWithContext, getSemanticVersion, getSetterFor, getProvidedCapabilities, isPolymorphicPullMethod, hasPublicAPIs, isAllowFocusMethod, hasAllowFocusMethods, createPolymorphicMethods, isExcludedMethod, isCallsMetricsMethod, getProvidedInterfaces, getUnidirectionalProviderInterfaceName } from '../shared/modules.mjs' +import { isRPCOnlyMethod, isProviderInterfaceMethod, getProviderInterface, getPayloadFromEvent, providerHasNoParameters, isTemporalSetMethod, hasMethodAttributes, getMethodAttributes, isEventMethodWithContext, getSemanticVersion, getSetterFor, getProvidedCapabilities, isPolymorphicPullMethod, hasPublicAPIs, isAllowFocusMethod, hasAllowFocusMethods, isExcludedMethod, isCallsMetricsMethod, getProvidedInterfaces, getUnidirectionalProviderInterfaceName } from '../shared/modules.mjs' import { extension, getNotifier, name as methodName, name, provides } from '../shared/methods.mjs' import isEmpty from 'crocks/core/isEmpty.js' -import { getReferencedSchema, getLinkedSchemaPaths, getSchemaConstraints, isSchema, localizeDependencies, isDefinitionReferencedBySchema, mergeAnyOf, mergeOneOf, getSafeEnumKeyName, getAllValuesForName } from '../shared/json-schema.mjs' +import { getReferencedSchema, getLinkedSchemaPaths, getSchemaConstraints, isSchema, localizeDependencies, isDefinitionReferencedBySchema, getSafeEnumKeyName, getAllValuesForName } from '../shared/json-schema.mjs' import Types from './types.mjs' @@ -50,7 +50,6 @@ let config = { extractSubSchemas: false, unwrapResultObjects: false, excludeDeclarations: false, - extractProviderSchema: false, } const state = { @@ -468,22 +467,6 @@ const promoteAndNameSubSchemas = (server, client) => { } const generateMacros = (server, client, templates, languages, options = {}) => { - // TODO: figure out anyOfs/polymorphs on the client RPC. It can work for events, but not providers - if (options.createPolymorphicMethods) { - let methods = [] - server.methods && server.methods.forEach(method => { - let polymorphicMethods = createPolymorphicMethods(method, server) - if (polymorphicMethods.length > 1) { - polymorphicMethods.forEach(polymorphicMethod => { - methods.push(polymorphicMethod) - }) - } - else { - methods.push(method) - } - }) - server.methods = methods - } // for languages that don't support nested schemas, let's promote them to first-class schemas w/ titles if (config.extractSubSchemas) { server = promoteAndNameSubSchemas(server, client) @@ -511,6 +494,8 @@ const generateMacros = (server, client, templates, languages, options = {}) => { macros.callsMetrics = true } + let start = Date.now() + const unique = list => list.map((item, i) => Object.assign(item, { index: i })).filter( (item, i, list) => !(list.find(x => x.name === item.name) && list.find(x => x.name === item.name).index < item.index)) Array.from(new Set(['types'].concat(config.additionalSchemaTemplates))).filter(dir => dir).forEach(dir => { @@ -522,11 +507,17 @@ const generateMacros = (server, client, templates, languages, options = {}) => { macros.enum_implementations[dir] = getTemplate('/sections/enums', templates).replace(/\$\{schema.list\}/g, schemasArray.filter(x => x.enum).map(s => s.impl).filter(body => body).join('\n')) }) + console.log(` - Generated types macros ${Date.now() - start}`) + start = Date.now() + state.typeTemplateDir = 'types' const imports = Object.fromEntries(Array.from(new Set(Object.keys(templates).filter(key => key.startsWith('/imports/')).map(key => key.split('.').pop()))).map(key => [key, generateImports(server, client, templates, { destination: key })])) const initialization = generateInitialization(server, client, templates) const eventsEnum = generateEvents(server, templates) + console.log(` - Generated imports, etc macros ${Date.now() - start}`) + start = Date.now() + const allMethodsArray = generateMethods(server, client, templates, languages, options.type) Array.from(new Set(['methods'].concat(config.additionalMethodTemplates))).filter(dir => dir).forEach(dir => { @@ -549,12 +540,18 @@ const generateMacros = (server, client, templates, languages, options = {}) => { } }) + console.log(` - Generated method macros ${Date.now() - start}`) + start = Date.now() + const xusesInterfaces = generateXUsesInterfaces(server, templates) const providerSubscribe = generateProviderSubscribe(server, client, templates, !!client) const providerInterfaces = generateProviderInterfaces(server, client, templates, 'interface', 'interfaces', !!client) const providerClasses = generateProviderInterfaces(server, client, templates, 'class', 'classes', !!client) const defaults = generateDefaults(server, client, templates) + console.log(` - Generated provider macros ${Date.now() - start}`) + start = Date.now() + const module = getTemplate('/codeblocks/module', templates) const moduleInclude = getTemplate('/codeblocks/module-include', templates) const moduleIncludePrivate = getTemplate('/codeblocks/module-include-private', templates) @@ -624,8 +621,6 @@ const insertMacros = (fContents = '', macros = {}) => { fContents = fContents.replace(/\$\{if\.enums\}(.*?)\$\{end\.if\.enums\}/gms, macros.enums.types.trim() ? '$1' : '') fContents = fContents.replace(/\$\{if\.declarations\}(.*?)\$\{end\.if\.declarations\}/gms, (macros.methods.declarations && macros.methods.declarations.trim() || macros.enums.types.trim()) || macros.types.types.trim()? '$1' : '') fContents = fContents.replace(/\$\{if\.callsmetrics\}(.*?)\$\{end\.if\.callsmetrics\}/gms, macros.callsMetrics ? '$1' : '') - fContents = fContents.replace(/\$\{if\.unidirectional\}(.*?)\$\{end\.if\.unidirectional\}/gms, macros.unidirectional ? '$1' : '') - fContents = fContents.replace(/\$\{if\.bidirectional\}(.*?)\$\{end\.if\.bidirectional\}/gms, !macros.unidirectional ? '$1' : '') fContents = fContents.replace(/\$\{module\.list\}/g, macros.module) fContents = fContents.replace(/\$\{module\.includes\}/g, macros.moduleInclude) @@ -719,6 +714,9 @@ const insertMacros = (fContents = '', macros = {}) => { fContents = fContents.replace(/\$\{if\.events\}.*?\$\{end\.if\.events\}/gms, '') } + fContents = fContents.replace(/\$\{if\.unidirectional\}(.*?)\$\{end\.if\.unidirectional\}/gms, macros.unidirectional ? '$1' : '') + fContents = fContents.replace(/\$\{if\.bidirectional\}(.*?)\$\{end\.if\.bidirectional\}/gms, !macros.unidirectional ? '$1' : '') + fContents = insertTableofContents(fContents) return fContents @@ -931,10 +929,11 @@ function generateSchemas(server, templates, options) { else { content = content.replace(/\$\{if\.description\}(.*?)\{end\.if\.description\}/gms, '$1') } + + const schemaShape = Types.getSchemaShape(schema, server, { templateDir: state.typeTemplateDir, primitive: config.primitives ? Object.keys(config.primitives).length > 0 : false, namespace: !config.copySchemasIntoModules }) const schemaImpl = Types.getSchemaShape(schema, server, { templateDir: state.typeTemplateDir, enumImpl: true, primitive: config.primitives ? Object.keys(config.primitives).length > 0 : false, namespace: !config.copySchemasIntoModules }) - content = content .replace(/\$\{schema.title\}/, (schema.title || name)) .replace(/\$\{schema.description\}/, schema.description || '') @@ -994,7 +993,14 @@ function generateSchemas(server, templates, options) { }) list = sortSchemasByReference(list) - list.forEach(item => generate(...item)) + list.forEach(item => { + try { + generate(...item) + } + catch (error) { + console.error(error) + } + }) return results } @@ -1282,8 +1288,8 @@ function generateMethods(server = {}, client = null, templates = {}, languages = } // TODO: this is called too many places... let's reduce that to just generateMethods -function insertMethodMacros(template, methodObj, server, client, templates, type = '', examples = [], languages = {}) { - try { +function insertMethodMacros(template, methodObj, server, client, templates, type = 'method', examples = [], languages = {}) { + // try { // need a guaranteed place to get client stuff from... const document = client || server const moduleName = getModuleName(server) @@ -1573,14 +1579,14 @@ function insertMethodMacros(template, methodObj, server, client, templates, type template = insertExampleMacros(template, examples || [], methodObj, server, templates) return template - } - catch (error) { - console.log(`Error processing method ${methodObj.name}`) - console.dir(methodObj) - console.log() - console.dir(error) - process.exit(1) - } + // } + // catch (error) { + // console.log(`Error processing method ${methodObj.name}`) + // console.dir(methodObj, { depth: 10 }) + // console.log() + // console.dir(error) + // process.exit(1) + // } } function insertExampleMacros(template, examples, method, json, templates) { @@ -1918,8 +1924,8 @@ function insertProviderInterfaceMacros(template, _interface, server = {}, client name: 'provider' }) - let type = config.templateExtensionMap && config.templateExtensionMap['methods'] && config.templateExtensionMap['methods'].includes(suffix) ? 'methods' : 'declarations' - return insertMethodMacros(interfaceDeclaration, method, server, client, templates, type) +// let type = config.templateExtensionMap && config.templateExtensionMap['methods'] && config.templateExtensionMap['methods'].includes(suffix) ? 'methods' : 'declarations' + return insertMethodMacros(interfaceDeclaration, method, server, client, templates) }).join('') + '\n') if (iface.length === 0) { diff --git a/src/macrofier/index.mjs b/src/macrofier/index.mjs index 01812d70..b40d7bb2 100644 --- a/src/macrofier/index.mjs +++ b/src/macrofier/index.mjs @@ -26,7 +26,7 @@ import { logHeader, logSuccess } from '../shared/io.mjs' import Types from './types.mjs' import path from 'path' import engine from './engine.mjs' -import { getLocalSchemas, replaceRef, replaceUri } from '../shared/json-schema.mjs' +import { findAll, flattenMultipleOfs, getLocalSchemas, replaceRef, replaceUri } from '../shared/json-schema.mjs' /************************************************************************************************/ /******************************************** MAIN **********************************************/ @@ -47,6 +47,7 @@ const macrofy = async ( templatesPerSchema, persistPermission, createPolymorphicMethods, + enableUnionTypes, createModuleDirectories, copySchemasIntoModules, mergeOnTitle, @@ -58,7 +59,6 @@ const macrofy = async ( additionalMethodTemplates, templateExtensionMap, excludeDeclarations, - extractProviderSchema, aggregateFiles, operators, primitives, @@ -79,6 +79,20 @@ const macrofy = async ( const serverRpc = await readJson(server) const clientRpc = client && await readJson(client) || null + // Combine all-ofs to make code-generation simplier + flattenMultipleOfs(serverRpc, 'allOf') + flattenMultipleOfs(clientRpc, 'allOf') + + // Combine union types (anyOf / oneOf) for languages that don't have them + // NOTE: anyOf and oneOf are both treated as ORs, i.e. oneOf is not XOR + if (!enableUnionTypes) { + flattenMultipleOfs(serverRpc, 'anyOf') + flattenMultipleOfs(serverRpc, 'oneOf') + + flattenMultipleOfs(clientRpc, 'anyOf') + flattenMultipleOfs(clientRpc, 'oneOf') + } + logHeader(`Generating ${headline} for version ${serverRpc.info.title} ${serverRpc.info.version}`) engine.setConfig({ @@ -93,7 +107,6 @@ const macrofy = async ( additionalMethodTemplates, templateExtensionMap, excludeDeclarations, - extractProviderSchema, operators }) @@ -208,6 +221,8 @@ const macrofy = async ( modules.forEach(module => { start = Date.now() const clientRpc2 = clientRpc && getClientModule(module.info.title, clientRpc, module) + logSuccess(` - gotClientModule ${Date.now() - start}ms`) + start = Date.now() const macros = engine.generateMacros(module, clientRpc2, templates, exampleTemplates, {hideExcluded: hideExcluded, copySchemasIntoModules: copySchemasIntoModules, createPolymorphicMethods: createPolymorphicMethods, type: 'methods'}) logSuccess(`Generated macros for module ${module.info.title} (${Date.now() - start}ms)`) diff --git a/src/macrofier/types.mjs b/src/macrofier/types.mjs index 9bce999b..2901cb5b 100644 --- a/src/macrofier/types.mjs +++ b/src/macrofier/types.mjs @@ -17,7 +17,7 @@ */ import deepmerge from 'deepmerge' -import { getReferencedSchema, localizeDependencies, getSafeEnumKeyName } from '../shared/json-schema.mjs' +import { getReferencedSchema, localizeDependencies, getSafeEnumKeyName, schemaReferencesItself } from '../shared/json-schema.mjs' import path from "path" let convertTuplesToArraysOrObjects = false @@ -150,7 +150,7 @@ const getXSchemaGroupFromProperties = (schema, title, properties, group) => { const getXSchemaGroup = (schema, module) => { let group = module.info ? module.info.title : module.title let bundles = module.definitions || module.components.schemas - + if (schema.title && bundles) { Object.entries(bundles).filter(([key, s]) => s.$id).forEach(([id, bundle]) => { const title = bundle.title @@ -303,7 +303,6 @@ const insertObjectPatternPropertiesMacros = (content, schema, module, title, opt if (patternSchema) { const shape = getSchemaShape(patternSchema, module, options2) let type = getSchemaType(patternSchema, module, options2).trimEnd() - const propertyNames = localizeDependencies(schema, module).propertyNames content = content .replace(/\$\{shape\}/g, shape) @@ -323,7 +322,7 @@ const getIndents = level => level ? ' ' : '' const insertObjectMacros = (content, schema, module, title, property, options) => { const options2 = options ? JSON.parse(JSON.stringify(options)) : {} options2.parent = title - options2.parentLevel = options.parentLevel + options2.parentLevel = options.level options2.level = options.level + 1 options2.templateDir = options.templateDir ;(['properties', 'properties.register', 'properties.assign']).forEach(macro => { @@ -337,15 +336,10 @@ const insertObjectMacros = (content, schema, module, title, property, options) = const subProperty = getTemplate(path.join(options2.templateDir, 'sub-property/object')) options2.templateDir += subProperty ? '/sub-property' : '' const objSeparator = getTemplate(path.join(options2.templateDir, 'object-separator')) - if (localizedProp.type === 'array' || localizedProp.anyOf || localizedProp.oneOf || (typeof localizedProp.const === 'string')) { - options2.property = name - options2.required = schema.required - } else { - options2.property = options.property - options2.required = schema.required && schema.required.includes(name) - } - const schemaShape = indent + getSchemaShape(localizedProp, module, options2).replace(/\n/gms, '\n' + indent) - const type = getSchemaType(localizedProp, module, options2) + options2.property = name + options2.required = schema.required && schema.required.includes(name) //schema.required + const schemaShape = indent + getSchemaShape(prop, module, options2).replace(/\n/gms, '\n' + indent) + const type = getSchemaType(prop, module, options2) // don't push properties w/ unsupported types if (type) { const description = getSchemaDescription(prop, module) @@ -360,8 +354,8 @@ const insertObjectMacros = (content, schema, module, title, property, options) = .replace(/\$\{if\.summary\}(.*?)\$\{end\.if\.summary\}/gms, description ? '$1' : '') .replace(/\$\{summary\}/g, description ? description.split('\n')[0] : '') .replace(/\$\{delimiter\}(.*?)\$\{end.delimiter\}/gms, i === schema.properties.length - 1 ? '' : '$1') - .replace(/\$\{if\.optional\}(.*?)\$\{end\.if\.optional\}/gms, ((schema.required && schema.required.includes(name)) || (localizedProp.required && localizedProp.required === true)) ? '' : '$1') - .replace(/\$\{if\.non.optional\}(.*?)\$\{end\.if\.non.optional\}/gms, ((schema.required && schema.required.includes(name)) || (localizedProp.required && localizedProp.required === true)) ? '$1' : '') + .replace(/\$\{if\.optional\}(.*?)\$\{end\.if\.optional\}/gms, ((schema.required && schema.required.includes(name))) ? '' : '$1') + .replace(/\$\{if\.non.optional\}(.*?)\$\{end\.if\.non.optional\}/gms, ((schema.required && schema.required.includes(name))) ? '$1' : '') .replace(/\$\{if\.base\.optional\}(.*?)\$\{end\.if\.base\.optional\}/gms, options.required ? '' : '$1') .replace(/\$\{if\.non\.object\}(.*?)\$\{end\.if\.non\.object\}/gms, isObject(localizedProp) ? '' : '$1') .replace(/\$\{if\.non\.array\}(.*?)\$\{end\.if\.non\.array\}/gms, (localizedProp.type === 'array') ? '' : '$1') @@ -509,7 +503,8 @@ const insertPrimitiveMacros = (content, schema, module, name, templateDir) => { return content } -const insertAnyOfMacros = (content, schema, module, name, namespace) => { +// +const insertAnyOfMacros = (content, schema, module, namespace) => { const itemTemplate = content if (content.split('\n').find(line => line.includes("${type}"))) { content = schema.anyOf.map((item, i) => itemTemplate @@ -537,12 +532,16 @@ const sanitize = (schema) => { return result } -function getSchemaShape(schema = {}, module = {}, { templateDir = 'types', parent = '', property = '', required = false, parentLevel = 0, level = 0, summary, descriptions = true, enums = true, enumImpl = false, skipTitleOnce = false, array = false, primitive = false, type = false, namespace=true } = {}) { +function getSchemaShape(schema = {}, module = {}, { templateDir = 'types', parent = '', property = '', required = false, parentLevel = 0, level = 0, summary, descriptions = true, enums = true, enumImpl = false, skipTitleOnce = false, array = false, primitive = false, type = false, namespace=true } = {}) { schema = sanitize(schema) if (level === 0 && !schema.title && !primitive) { return '' } + if (schema.anyOf && schema.anyOf.find(s => s.$ref?.indexOf('/ListenResponse') >= 0)) { + schema = schema.anyOf.find(s => !s.$ref || s.$ref.indexOf('/ListenResponse') === -1) + } + const theTitle = insertSchemaMacros(getTemplate(path.join(templateDir, 'title')), schema, module, { name: schema.title, parent, property, required, recursive: false }) const moduleTitle = module.info ? module.info.title : module.title @@ -599,25 +598,16 @@ function getSchemaShape(schema = {}, module = {}, { templateDir = 'types', paren return insertSchemaMacros(result, schema, module, { name: theTitle, parent, property, required, templateDir }) } else if (schema.anyOf || schema.oneOf) { - const template = getTemplate(path.join(templateDir, 'anyOfSchemaShape')) - let shape - if (template) { - shape = insertAnyOfMacros(template, schema, module, theTitle, namespace) - } - else { - // borrow anyOf logic, note that schema is a copy, so we're not breaking it. - if (!schema.anyOf) { - schema.anyOf = schema.oneOf - } - shape = insertAnyOfMacros(getTemplate(path.join(templateDir, 'anyOf')) || genericTemplate, schema, module, theTitle, namespace) - } - if (shape) { - result = result.replace(/\$\{shape\}/g, shape) - return insertSchemaMacros(result, schema, module, { name: theTitle, parent, property, required }) - } - else { - return '' + // borrow anyOf logic, note that schema is a copy, so we're not breaking it. + if (!schema.anyOf) { + schema.anyOf = schema.oneOf } + + let template = getTemplate(path.join(templateDir, 'anyOf')) || genericTemplate + template = insertAnyOfMacros(template, schema, module, namespace) + + result = result.replace(/\$\{shape\}/g, template) + return insertSchemaMacros(result, schema, module, { name: theTitle, parent, property, required }) } else if (schema.allOf) { const merger = (key) => function (a, b) { @@ -849,8 +839,8 @@ function getSchemaType(schema, module, { templateDir = 'types', link = false, co if (!schema.anyOf) { schema.anyOf = schema.oneOf } - // todo... we probably shouldn't allow untitled anyOfs, at least not w/out a feature flag - const shape = insertAnyOfMacros(getTemplate(path.join(templateDir, 'anyOf')), schema, module, theTitle, namespace) + + const shape = insertAnyOfMacros(getTemplate(path.join(templateDir, 'anyOf')), schema, module, namespace) return insertSchemaMacros(shape, schema, module, { name: theTitle, recursive: false }) diff --git a/src/openrpc/index.mjs b/src/openrpc/index.mjs index 94e5f5da..e3d932df 100644 --- a/src/openrpc/index.mjs +++ b/src/openrpc/index.mjs @@ -20,7 +20,7 @@ import { readJson, readFiles, readDir, writeJson } from "../shared/filesystem.mj import { addExternalMarkdown, addExternalSchemas, fireboltize } from "../shared/modules.mjs" import path from "path" import { logHeader, logSuccess } from "../shared/io.mjs" -import { namespaceRefs } from "../shared/json-schema.mjs" +import { flattenMultipleOfs, namespaceRefs } from "../shared/json-schema.mjs" const run = async ({ input: input, diff --git a/src/sdk/index.mjs b/src/sdk/index.mjs index 8c2f1380..91228433 100755 --- a/src/sdk/index.mjs +++ b/src/sdk/index.mjs @@ -81,7 +81,8 @@ const run = async ({ templatesPerModule: config.language.templatesPerModule, templatesPerSchema: config.language.templatesPerSchema, persistPermission: config.language.persistPermission, - createPolymorphicMethods: config.language.createPolymorphicMethods, + createPolymorphicMethods: config.language.createPolymorphicMethods || false, + enableUnionTypes: config.language.enableUnionTypes || false, operators: config.language.operators, primitives: config.language.primitives, createModuleDirectories: config.language.createModuleDirectories, @@ -95,7 +96,6 @@ const run = async ({ additionalMethodTemplates: config.language.additionalMethodTemplates, templateExtensionMap: config.language.templateExtensionMap, excludeDeclarations: config.language.excludeDeclarations, - extractProviderSchema: config.language.extractProviderSchema, staticModuleNames: staticModuleNames, hideExcluded: true, moduleWhitelist: moduleWhitelist, diff --git a/src/shared/json-schema.mjs b/src/shared/json-schema.mjs index ab81541c..39837f32 100644 --- a/src/shared/json-schema.mjs +++ b/src/shared/json-schema.mjs @@ -212,12 +212,12 @@ const getPropertySchema = (json, dotPath, document) => { for (var i=0; i j >= i ).join('.') - if (node.$ref) { - node = getPropertySchema(getReferencedSchema(node.$ref, document), remainingPath, document) - } - else if (property === '') { + if (property === '') { return node } + else if (node.$ref) { + node = getPropertySchema(getReferencedSchema(node.$ref, document), remainingPath, document) + } else if (node.type === 'object' || (node.type && node.type.includes && node.type.includes('object'))) { if (node.properties && node.properties[property]) { node = node.properties[property] @@ -263,9 +263,10 @@ const getPropertiesInSchema = (json, document) => { props.push(...Object.keys(node.properties)) } - if (node.propertyNames) { - props.push(...node.propertyNames) - } + // TODO: this propertyNames requires either additionalProperties or patternProperties in order to use this method w/ getPropertySchema, as intended... + // if (node.propertyNames) { + // props.push(...node.propertyNames) + // } return props } @@ -457,13 +458,7 @@ const localizeDependencies = (json, document, schemas = {}, options = defaultLoc findAndMergeAllOfs(pointer[key]) } else if (key === 'allOf' && Array.isArray(pointer[key])) { - const union = deepmerge.all(pointer.allOf.reverse()) // reversing so lower `title` attributes will win - const title = pointer.title - Object.assign(pointer, union) - if (title) { - pointer.title = title - } - delete pointer.allOf + definition = mergeAllOf(pointer) } }) } @@ -474,6 +469,19 @@ const localizeDependencies = (json, document, schemas = {}, options = defaultLoc return definition } +const mergeAllOf = (schema) => { + if (schema.allOf) { + const union = deepmerge.all(schema.allOf.reverse()) // reversing so lower `title` attributes will win + const title = schema.title + Object.assign(schema, union) + if (title) { + schema.title = title + } + delete schema.allOf + } + return schema +} + const getLocalSchemas = (json = {}) => { return Array.from(new Set(getLocalSchemaPaths(json).map(path => getPathOr(null, path, json)))) } @@ -493,78 +501,441 @@ const isDefinitionReferencedBySchema = (name = '', moduleJson = {}) => { return (refs.length > 0) } -function union(schemas) { - - const result = {}; - for (const schema of schemas) { - for (const [key, value] of Object.entries(schema)) { - if (!result.hasOwnProperty(key)) { - // If the key does not already exist in the result schema, add it - if (value && value.anyOf) { - result[key] = union(value.anyOf) - } else if (key === 'title' || key === 'description' || key === 'required') { - //console.warn(`Ignoring "${key}"`) - } else { - result[key] = value; - } - } else if (key === '$ref') { - if (result[key].endsWith("/ListenResponse")) { - - } - // If the key is '$ref' make sure it's the same - else if(result[key] === value) { - //console.warn(`Ignoring "${key}" that is already present and same`) - } else { - console.warn(`ERROR "${key}" is not same -${JSON.stringify(result, null, 4)} ${key} ${result[key]} - ${value}`); - throw "ERROR: $ref is not same" - } - } else if (key === 'type') { - // If the key is 'type', merge the types of the two schemas - if(result[key] === value) { - //console.warn(`Ignoring "${key}" that is already present and same`) - } else { - console.warn(`ERROR "${key}" is not same -${JSON.stringify(result, null, 4)} ${key} ${result[key]} - ${value}`); - throw "ERROR: type is not same" - } - } else { - //If the Key is a const then merge them into an enum - if(value && value.const) { - if(result[key].enum) { - result[key].enum = Array.from(new Set([...result[key].enum, value.const])) +const findAll = (document, finder) => { + const results = [] + + if (document && finder(document)) { + results.push(document) + } + + if ((typeof document) !== 'object' || !document) { + return results + } + + Object.keys(document).forEach(key => { + + if (Array.isArray(document) && key === 'length') { + return results + } + else if (typeof document[key] === 'object') { + results.push(...findAll(document[key], finder)) + } + }) + + return results +} + +const flattenMultipleOfs = (document, type, pointer, path) => { + if (!pointer) { + pointer = document + path = '' + } + + if ((typeof pointer) !== 'object' || !pointer) { + return + } + + if (pointer !== document && schemaReferencesItself(pointer, path.split('.'))) { + console.warn(`Skipping recursive schema: ${pointer.title}`) + return + } + + Object.keys(pointer).forEach(key => { + + if (Array.isArray(pointer) && key === 'length') { + return + } + if ( (pointer.$id && pointer !== document) || ((key !== type) && (typeof pointer[key] === 'object') && (pointer[key] != null))) { + flattenMultipleOfs(document, type, pointer[key], path + '.' + key) + } + else if (key === type && Array.isArray(pointer[key])) { + + try { + const schemas = pointer[key] + if (schemas.find(schema => schema.$ref?.endsWith("/ListenResponse"))) { + // ignore the ListenResponse parent anyOf, but dive into it's sibling + const sibling = schemas.find(schema => !schema.$ref?.endsWith("/ListenResponse")) + const n = schemas.indexOf(sibling) + flattenMultipleOfs(document, type, schemas[n], path + '.' + key + '.' + n) + } + else { + const title = pointer.title + let debug = false + Object.assign(pointer, combineSchemas(pointer[key], document, path, type === 'allOf')) + if (title) { + pointer.title = title + } + delete pointer[key] + } + } + catch(error) { + console.warn(` - Unable to flatten ${type} in ${path}`) + console.log(error) + } + } + }) +} + +function combineProperty(result, schema, document, prop, path, all) { + if (result.properties === undefined || result.properties[prop] === undefined) { + if (result.additionalProperties === false || schema.additionalProperties === false) { + if (all) { + // leave it out + } + else { + result.properties = result.properties || {} + result.properties[prop] = getPropertySchema(schema, prop, document) + } + } + else if (typeof result.additionalProperties === 'object') { + result.properties = result.properties || {} + result.properties[prop] = combineSchemas([result.additionalProperties, getPropertySchema(schema, prop, document)], document, path + '.' + prop, all) + } + else { + result.properties = result.properties || {} + result.properties[prop] = getPropertySchema(schema, prop, document) + } + } + else if (schema.properties === undefined || schema.properties[prop] === undefined) { + if (result.additionalProperties === false || schema.additionalProperties === false) { + if (all) { + delete result.properties[prop] + } + else { + // leave it + } + } + else if (typeof schema.additionalProperties === 'object') { + result.properties = result.properties || {} + result.properties[prop] = combineSchemas([schema.additionalProperties, getPropertySchema(result, prop, document)], document, path + '.' + prop, all) + } + else { + // do nothing + } + } + else { + const a = getPropertySchema(result, prop, document) + const b = getPropertySchema(schema, prop, document) + + result.properties[prop] = combineSchemas([a, b], document, path + '.' + prop, all, true) + } + + result = JSON.parse(JSON.stringify(result)) +} + +// TODO: fix titles, drop if/then/else/not +function combineSchemas(schemas, document, path, all, createRefIfNeeded=false) { + schemas = JSON.parse(JSON.stringify(schemas)) + let createRefSchema = false + + if (createRefIfNeeded && schemas.find(s => s?.$ref) && !schemas.every(s => s.$ref === schemas.find(s => s?.$ref).$ref)) { + createRefSchema = true + } + + const reference = createRefSchema ? schemas.filter(schema => schema?.$ref).map(schema => schema.$ref).reduce( (prefix, ref, i, arr) => { + if (prefix === '') { + if (arr.length === 1) { + return ref.split('/').slice(0, -1).join('/') + '/' + } + else { + return ref + } + } + else { + let index = 0 + while ((index < Math.min(prefix.length, ref.length)) && (prefix.charAt(index) === ref.charAt(index))) { + index++ + } + return prefix.substring(0, index) + } + }, '') : '' + + const resolve = (schema) => { + while (schema.$ref) { + if (!getReferencedSchema(schema.$ref, document)) { + console.log(`getReferencedSChema returned null`) + console.dir(schema) + } + schema = getReferencedSchema(schema.$ref, document) + } + return schema + } + + let debug = false + + const merge = (schema) => { + if (schema.allOf) { + schema.allOf = schema.allOf.map(resolve) + Object.assign(schema, combineSchemas(schema.allOf, document, path, true)) + delete schema.allOf + } + if (schema.oneOf) { + schema.oneOf = schema.oneOf.map(resolve) + Object.assign(schema, combineSchemas(schema.oneOf, document, path, false)) + delete schema.oneOf + } + if (schema.anyOf) { + schema.anyOf = schema.anyOf.map(resolve) + Object.assign(schema, combineSchemas(schema.anyOf, document, path, false)) + delete schema.anyOf + } + return schema + } + + const flatten = (schema) => { + while (schema.$ref || schema.oneOf || schema.anyOf || schema.allOf) { + schema = resolve(schema) + schema = merge(schema) + } + return schema + } + + let result = schemas.shift() + + schemas.forEach(schema => { + + if (!schema) { + return // skip + } + + if (schema.$ref && (schema.$ref === result.$ref)) { + return + } + + result = JSON.parse(JSON.stringify(flatten(result))) + schema = JSON.parse(JSON.stringify(flatten(schema))) + + if (schema.examples && result.examples) { + result.examples.push(...schema.examples) + } + + if (schema.anyOf) { + throw "Cannot combine schemas that contain anyOf" + } + else if (schema.oneOf) { + throw "Cannot combine schemas that contain oneOf" + } + else if (schema.allOf) { + throw "Cannot combine schemas that contain allOf" + } + else if (Array.isArray(schema.type)) { + throw "Cannot combine schemas that have type set to an Array" + } + else { + if (result.const !== undefined && schema.const != undefined) { + if (result.const === schema.const) { + return + } + else if (all) { + throw `Combined allOf resulted in impossible schema: const ${schema.const} !== const ${result.const}` + } + else { + result.enum = [result.const, schema.const] + result.type = typeof result.const + delete result.const + } + } + else if (result.enum && schema.enum) { + if (all) { + result.enum = result.enum.filter(value => schema.enum.includes(value)) + if (result.enum.length === 0) { + throw `Combined allOf resulted in impossible schema: enum: []` + } + } + else { + result.enum = Array.from(new Set(result.enum.concat(schema.enum))) + } + } + else if ((result.const !== undefined || schema.const !== undefined) && (result.enum || schema.enum)) { + if (all) { + const c = result.const !== undefined ? result.const : schema.const + const e = result.enum || schema.enum + if (e.contains(c)) { + result.const = c + delete result.enum + delete result.type + } + else { + throw `Combined allOf resulted in impossible schema: enum: ${e} does not contain const: ${c}` + } + } + else { + result.enum = Array.from(new Set([].concat(result.enum || result.const).concat(schema.enum || schema.const))) + result.type = result.type || schema.type + delete result.const + } + } + else if ((result.const !== undefined || schema.const !== undefined) && (result.type || schema.type)) { + // TODO need to make sure the types match + if (all) { + result.const = result.const !== undefined ? result.const : schema.const + delete result.type + } + else { + result.type = result.type || schema.type + delete result.const + } + } + else if (schema.type !== result.type) { + throw `Cannot combine schemas with property type conflicts, '${path}': ${schema.type} != ${result.type} in ${schema.title} / ${result.title}` + } + else if ((result.enum || schema.enum) && (result.type || schema.type)) { + if (all) { + result.enum = result.enum || schema.enum + } + else { + result.type = result.type || schema.type + delete result.enum + } + } + else if (schema.type === "object") { + const propsInSchema = getPropertiesInSchema(schema, document) + const propsOnlyInResult = getPropertiesInSchema(result, document).filter(p => !propsInSchema.includes(p)) + + propsInSchema.forEach(prop => { + combineProperty(result, schema, document, prop, path, all) + delete result.title + }) + + propsOnlyInResult.forEach(prop => { + combineProperty(result, schema, document, prop, path, all) + delete result.title + }) + + if (result.additionalProperties === false || schema.additionalProperties === false) { + if (all) { + result.additionalProperties = false + } + else { + if (result.additionalProperties === true || schema.additionalProperties === true || result.additionalProperties === undefined || schema.additionalProperties === undefined) { + result.additionalProperties = true + } + else if (typeof result.additionalProperties === 'object' || typeof schema.additionalProperties === 'object') { + result.additionalProperties = result.additionalProperties || schema.additionalProperties + } + } + } + else if (typeof result.additionalProperties === 'object' || typeof schema.additionalProperties === 'object') { + result.additionalProperties = combineSchemas([result.additionalProperties, schema.additionalProperties], document, path, all) + } + + if (Array.isArray(result.propertyNames) && Array.isArray(schema.propertyNames)) { + if (all) { + result.propertyNames = Array.from(new Set(result.propertyNames.concat(schema.propertyNames))) + } + else { + result.propertyNames = result.propertyNames.filter(prop => schema.propertyNames.includes(prop)) + } + } + else if (Array.isArray(result.propertyNames) || Array.isArray(schema.propertyNames)) { + if (all) { + result.propertyNames = result.propertyNames || schema.propertyNames + } + else { + delete result.propertyNames + } + } + + if (result.patternProperties || schema.patternProperties) { + throw `Cannot combine object schemas that have patternProperties ${schema.title} / ${result.title}, ${path}` + } + + if (result.required && schema.required) { + if (all) { + result.required = Array.from(new Set(result.required.concat(schema.required))) } else { - result[key].enum = Array.from(new Set([result[key].const, value.const])) - delete result[key].const + result.required = result.required.filter(prop => schema.required.includes(prop)) } } - // If the key exists in both schemas and is not 'type', merge the values - else if (Array.isArray(result[key])) { - // If the value is an array, concatenate the arrays and remove duplicates - result[key] = Array.from(new Set([...result[key], ...value])) - } else if (result[key] && result[key].enum && value && value.enum) { - //If the value is an enum, merge the enums together and remove duplicates - result[key].enum = Array.from(new Set([...result[key].enum, ...value.enum])) - } else if (typeof result[key] === 'object' && typeof value === 'object') { - // If the value is an object, recursively merge the objects - result[key] = union([result[key], value]); - } else if (result[key] !== value) { - // If the value is a primitive and is not the same in both schemas, ignore it - //console.warn(`Ignoring conflicting value for key "${key}"`) + else if (result.required || schema.required) { + if (all) { + result.required = result.required || schema.required + } + else { + delete result.required + } } } + else if (schema.type === "array") { + if (Array.isArray(result.items) || Array.isArray(schema.items)) { + throw `Cannot combine tuple schemas, ${path}: ${schema.title} / ${result.title}` + } + result.items = combineSchemas([result.items, schema.items], document, path, all) + } + + if (result.title || schema.title) { + result.title = schema.title || result.title // prefer titles from lower in the any/all/oneOf list + } + + // combine all other stuff + const skip = ['title', 'type', '$ref', 'const', 'enum', 'properties', 'items', 'additionalProperties', 'patternProperties', 'anyOf', 'oneOf', 'allOf'] + const keysInSchema = Object.keys(schema) + const keysOnlyInResult = Object.keys(result).filter(k => !keysInSchema.includes(k)) + + keysInSchema.filter(key => !skip.includes(key)).forEach(key => { + if (result[key] === undefined) { + if (all) { + result[key] = schema[key] + } + } + else { + // not worth doing this for code-generation, e.g. minimum doesn't actually affect type defintions in most languages + } + }) + + keysOnlyInResult.filter(key => !skip.includes(key)).forEach(key => { + if (all) { + // do nothing + } + else { + delete result[key] + } + }) } - } - return result; -} + }) -function mergeAnyOf(schema) { - return union(schema.anyOf) -} + delete result.if + delete result.then + delete result.else + delete result.not -function mergeOneOf(schema) { - return union(schema.oneOf) + if (reference && createRefSchema) { + const [fragment, uri] = reference.split('#').reverse() + const title = result.title || path.split('.').slice(-2).map(x => x.charAt(0).toUpperCase() + x.substring(1)).join('') + + result.title = title + + let bundle + + if (uri) { + bundle = findAll(document, s => s.$id === uri)[0] + } + else { + bundle = document + } + + let pathArray = (fragment + title).split('/') + const name = pathArray.pop() + let key, i=1 + while (key = pathArray[i]) { + bundle = bundle[key] + i++ + } + + bundle[name] = result + + const refSchema = { + $ref: [uri ? uri : '', [...pathArray, name].join('/')].join('#') + } + + return refSchema + } + + return result } + const getSafeEnumKeyName = (value) => value.split(':').pop() // use last portion of urn:style:values .replace(/[\.\-]/g, '_') // replace dots and dashes .replace(/\+/g, '_plus') // change + to _plus @@ -592,6 +963,8 @@ export { replaceRef, namespaceRefs, removeIgnoredAdditionalItems, - mergeAnyOf, - mergeOneOf + combineSchemas, + flattenMultipleOfs, + schemaReferencesItself, + findAll } diff --git a/src/shared/modules.mjs b/src/shared/modules.mjs index 333ab8f4..e4d1d804 100644 --- a/src/shared/modules.mjs +++ b/src/shared/modules.mjs @@ -390,7 +390,6 @@ const getPayloadFromEvent = (event, client) => { } } catch (error) { - m(event) throw error } } @@ -1423,138 +1422,6 @@ const generateEventListenResponse = json => { return json } -const getAnyOfSchema = (inType, json) => { - let anyOfTypes = [] - let outType = localizeDependencies(inType, json) - if (outType.schema.anyOf) { - let definition = '' - if (inType.schema['$ref'] && (inType.schema['$ref'][0] === '#')) { - definition = getReferencedSchema(inType.schema['$ref'], json, json['x-schemas']) - } - else { - definition = outType.schema - } - definition.anyOf.forEach(anyOf => { - anyOfTypes.push(anyOf) - }) - outType.schema.anyOf = anyOfTypes - } - return outType -} - -const generateAnyOfSchema = (anyOf, name, summary) => { - let anyOfType = {} - anyOfType["name"] = name; - anyOfType["summary"] = summary - anyOfType["schema"] = anyOf - return anyOfType -} - -const generateParamsAnyOfSchema = (methodParams, anyOf, anyOfTypes, title, summary) => { - let params = [] - methodParams.forEach(p => { - if (p.schema.anyOf === anyOfTypes) { - let anyOfType = generateAnyOfSchema(anyOf, p.name, summary) - anyOfType.required = p.required - params.push(anyOfType) - } - else { - params.push(p) - } - }) - return params -} - -const generateResultAnyOfSchema = (method, methodResult, anyOf, anyOfTypes, title, summary) => { - let methodResultSchema = {} - if (methodResult.schema.anyOf === anyOfTypes) { - let anyOfType = generateAnyOfSchema(anyOf, title, summary) - let index = 0 - if (isEventMethod(method)) { - index = (method.result.schema.anyOf || method.result.schema.oneOf).indexOf(getPayloadFromEvent(method)) - } - else { - index = (method.result.schema.anyOf || method.result.schema.oneOf).indexOf(anyOfType) - } - if (method.result.schema.anyOf) { - methodResultSchema["anyOf"] = Object.assign([], method.result.schema.anyOf) - methodResultSchema.anyOf[index] = anyOfType.schema - } - else if (method.result.schema.oneOf) { - methodResultSchema["oneOf"] = Object.assign([], method.result.schema.oneOf) - methodResultSchema.oneOf[index] = anyOfType.schema - } - else { - methodResultSchema = anyOfType.schema - } - } - return methodResultSchema -} - -const createPolymorphicMethods = (method, json) => { - let anyOfTypes - let methodParams = [] - let methodResult = Object.assign({}, method.result) - - method.params.forEach(p => { - if (p.schema) { - let param = getAnyOfSchema(p, json) - if (param.schema.anyOf && anyOfTypes) { - //anyOf is allowed with only one param in the params list - throw `WARNING anyOf is repeated with param:${p}` - } - else if (param.schema.anyOf) { - anyOfTypes = param.schema.anyOf - } - methodParams.push(param) - } - }) - let foundAnyOfParams = anyOfTypes ? true : false - - if (isEventMethod(method)) { - methodResult.schema = getPayloadFromEvent(method) - } - methodResult = getAnyOfSchema(methodResult, json) - let foundAnyOfResult = methodResult.schema.anyOf ? true : false - if (foundAnyOfParams === true && foundAnyOfResult === true) { - throw `WARNING anyOf is already with param schema, it is repeated with ${method.name} result too` - } - else if (foundAnyOfResult === true) { - anyOfTypes = methodResult.schema.anyOf - } - let polymorphicMethodSchemas = [] - //anyOfTypes will be allowed either in any one of the params or in result - if (anyOfTypes) { - let polymorphicMethodSchema = { - name: {}, - tags: {}, - summary: `${method.summary}`, - params: {}, - result: {}, - examples: {} - } - anyOfTypes.forEach(anyOf => { - - let localized = localizeDependencies(anyOf, json) - let title = localized.title || localized.name || '' - let summary = localized.summary || localized.description || '' - polymorphicMethodSchema.rpc_name = method.name - polymorphicMethodSchema.name = foundAnyOfResult && isEventMethod(method) ? `${method.name}${title}` : method.name - polymorphicMethodSchema.tags = method.tags - polymorphicMethodSchema.params = foundAnyOfParams ? generateParamsAnyOfSchema(methodParams, anyOf, anyOfTypes, title, summary) : methodParams - polymorphicMethodSchema.result = Object.assign({}, method.result) - polymorphicMethodSchema.result.schema = foundAnyOfResult ? generateResultAnyOfSchema(method, methodResult, anyOf, anyOfTypes, title, summary) : methodResult.schema - polymorphicMethodSchema.examples = method.examples - polymorphicMethodSchemas.push(Object.assign({}, polymorphicMethodSchema)) - }) - } - else { - polymorphicMethodSchemas = method - } - - return polymorphicMethodSchemas -} - const isSubSchema = (schema) => schema.type === 'object' || (schema.type === 'string' && schema.enum) const isSubEnumOfArraySchema = (schema) => (schema.type === 'array' && schema.items.enum) @@ -1760,38 +1627,41 @@ const addExternalSchemas = (json, sharedSchemas) => { // TODO: make this recursive, and check for group vs schema const removeUnusedSchemas = (json) => { const schema = JSON.parse(JSON.stringify(json)) + const components = schema.components + schema.components = { schemas: {} } + const refs = getAllValuesForName('$ref', schema) - const recurse = (schema, path) => { - let deleted = false - Object.keys(schema).forEach(name => { - if (isSchema(schema[name])) { - const used = refs.includes(path + '/' + name) || ((name.startsWith('https://') && refs.find(ref => ref.startsWith(name)))) //isDefinitionReferencedBySchema(path + '/' + name, json) - if (!used) { - delete schema[name] - deleted = true - } - else { + const addSchemas = (schema, refs) => { + let added = false + refs.forEach(ref => { + if (ref.startsWith("https://")) { + const [uri, fragment] = ref.split("#") + if (!schema.components.schemas[uri]) { + schema.components.schemas[uri] = components.schemas[uri] + console.log(`Adding ${uri}`) + added = true } } - else if (typeof schema[name] === 'object') { - deleted = deleted || recurse(schema[name], path + '/' + name) + else { + const key = ref.split("/").pop() + if (!schema.components.schemas[key]) { + schema.components.schemas[key] = components.schemas[key] + console.log(`Adding ${key}`) + added = true + } } }) - return deleted + return added } if (schema.components.schemas) { - while(recurse(schema.components.schemas, '#/components/schemas')) { + while(addSchemas(schema, refs)) { refs.length = 0 refs.push(...getAllValuesForName('$ref', schema)) } } - if (schema['x-schemas']) { - while(recurse(schema['x-schemas'], '#/x-schemas')) {} - } - return schema } @@ -2006,6 +1876,5 @@ export { getSemanticVersion, addExternalMarkdown, addExternalSchemas, - getExternalMarkdownPaths, - createPolymorphicMethods + getExternalMarkdownPaths }