From f3b5a6d8c024a4082eab92dcb8f58dd0b97040b8 Mon Sep 17 00:00:00 2001 From: Timo Stamm Date: Tue, 27 Feb 2024 00:09:28 +0100 Subject: [PATCH] Add "dependencies" to DescFile (#727) --- .../protobuf-test/src/descriptor-set.test.ts | 127 +++++++++++------- .../protobuf/src/create-descriptor-set.ts | 33 +++-- packages/protobuf/src/descriptor-set.ts | 20 +-- 3 files changed, 114 insertions(+), 66 deletions(-) diff --git a/packages/protobuf-test/src/descriptor-set.test.ts b/packages/protobuf-test/src/descriptor-set.test.ts index 200c0d2c9..e25b4458a 100644 --- a/packages/protobuf-test/src/descriptor-set.test.ts +++ b/packages/protobuf-test/src/descriptor-set.test.ts @@ -50,56 +50,74 @@ import { join } from "node:path"; const fdsBytes = readFileSync("./descriptorset.binpb"); describe("DescriptorSet", () => { - const set = createDescriptorSet(fdsBytes); - test("proto2 syntax", () => { - const descFile = set.files.find((f) => f.name == "extra/proto2"); - expect(descFile).toBeDefined(); - expect(descFile?.syntax).toBe("proto2"); - expect(descFile?.edition).toBe(Edition.EDITION_PROTO2); - expect(descFile?.getFeatures()).toStrictEqual( - new FeatureSet({ - fieldPresence: FeatureSet_FieldPresence.EXPLICIT, - enumType: FeatureSet_EnumType.CLOSED, - repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.EXPANDED, - utf8Validation: FeatureSet_Utf8Validation.NONE, - messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED, - jsonFormat: FeatureSet_JsonFormat.LEGACY_BEST_EFFORT, - }), - ); - }); - test("proto3 syntax", () => { - const descFile = set.files.find((f) => f.name == "extra/proto3"); - expect(descFile).toBeDefined(); - expect(descFile?.syntax).toBe("proto3"); - expect(descFile?.edition).toBe(Edition.EDITION_PROTO3); - expect(descFile?.getFeatures()).toStrictEqual( - new FeatureSet({ - fieldPresence: FeatureSet_FieldPresence.IMPLICIT, - enumType: FeatureSet_EnumType.OPEN, - repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED, - utf8Validation: FeatureSet_Utf8Validation.VERIFY, - messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED, - jsonFormat: FeatureSet_JsonFormat.ALLOW, - }), - ); - }); - test("edition 2023", () => { - const descFile = set.files.find( - (f) => f.name == "editions/edition2023-default-features", - ); - expect(descFile).toBeDefined(); - expect(descFile?.syntax).toBe("editions"); - expect(descFile?.edition).toBe(Edition.EDITION_2023); - expect(descFile?.getFeatures()).toStrictEqual( - new FeatureSet({ - fieldPresence: FeatureSet_FieldPresence.EXPLICIT, - enumType: FeatureSet_EnumType.OPEN, - repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED, - utf8Validation: FeatureSet_Utf8Validation.VERIFY, - messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED, - jsonFormat: FeatureSet_JsonFormat.ALLOW, - }), - ); + describe("file", () => { + test("proto2 syntax", () => { + const set = createDescriptorSet(fdsBytes); + const descFile = set.files.find((f) => f.name == "extra/proto2"); + expect(descFile).toBeDefined(); + expect(descFile?.syntax).toBe("proto2"); + expect(descFile?.edition).toBe(Edition.EDITION_PROTO2); + expect(descFile?.getFeatures()).toStrictEqual( + new FeatureSet({ + fieldPresence: FeatureSet_FieldPresence.EXPLICIT, + enumType: FeatureSet_EnumType.CLOSED, + repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.EXPANDED, + utf8Validation: FeatureSet_Utf8Validation.NONE, + messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED, + jsonFormat: FeatureSet_JsonFormat.LEGACY_BEST_EFFORT, + }), + ); + }); + test("proto3 syntax", () => { + const set = createDescriptorSet(fdsBytes); + const descFile = set.files.find((f) => f.name == "extra/proto3"); + expect(descFile).toBeDefined(); + expect(descFile?.syntax).toBe("proto3"); + expect(descFile?.edition).toBe(Edition.EDITION_PROTO3); + expect(descFile?.getFeatures()).toStrictEqual( + new FeatureSet({ + fieldPresence: FeatureSet_FieldPresence.IMPLICIT, + enumType: FeatureSet_EnumType.OPEN, + repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED, + utf8Validation: FeatureSet_Utf8Validation.VERIFY, + messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED, + jsonFormat: FeatureSet_JsonFormat.ALLOW, + }), + ); + }); + test("edition 2023", () => { + const set = createDescriptorSet(fdsBytes); + const descFile = set.files.find( + (f) => f.name == "editions/edition2023-default-features", + ); + expect(descFile).toBeDefined(); + expect(descFile?.syntax).toBe("editions"); + expect(descFile?.edition).toBe(Edition.EDITION_2023); + expect(descFile?.getFeatures()).toStrictEqual( + new FeatureSet({ + fieldPresence: FeatureSet_FieldPresence.EXPLICIT, + enumType: FeatureSet_EnumType.OPEN, + repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED, + utf8Validation: FeatureSet_Utf8Validation.VERIFY, + messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED, + jsonFormat: FeatureSet_JsonFormat.ALLOW, + }), + ); + }); + test("dependencies", async () => { + const fdsBin = await new UpstreamProtobuf().compileToDescriptorSet({ + "a.proto": `syntax="proto3"; + import "b.proto"; + import "c.proto";`, + "b.proto": `syntax="proto3";`, + "c.proto": `syntax="proto3";`, + }); + const set = createDescriptorSet(fdsBin); + const a = set.files[2]; + expect(a.name).toBe("a"); + expect(a.dependencies.length).toBe(2); + expect(a.dependencies.map((f) => f.name)).toStrictEqual(["b", "c"]); + }); }); describe("edition feature options", () => { test("file options should apply to all elements", async () => { @@ -423,6 +441,7 @@ describe("DescriptorSet", () => { }); }); test("knows extension", () => { + const set = createDescriptorSet(fdsBytes); const ext = set.extensions.get( "protobuf_unittest.optional_int32_extension", ); @@ -442,6 +461,7 @@ describe("DescriptorSet", () => { ); }); test("knows nested extension", () => { + const set = createDescriptorSet(fdsBytes); const ext = set.extensions.get( "protobuf_unittest.TestNestedExtension.nested_string_extension", ); @@ -465,6 +485,7 @@ describe("DescriptorSet", () => { }); describe("declarationString()", () => { test("for field with options", () => { + const set = createDescriptorSet(fdsBytes); const message = set.messages.get(JsonNamesMessage.typeName); expect(message).toBeDefined(); if (message !== undefined) { @@ -475,6 +496,7 @@ describe("DescriptorSet", () => { } }); test("for field with labels", () => { + const set = createDescriptorSet(fdsBytes); const message = set.messages.get(RepeatedScalarValuesMessage.typeName); expect(message).toBeDefined(); if (message !== undefined) { @@ -485,6 +507,7 @@ describe("DescriptorSet", () => { } }); test("for map field", () => { + const set = createDescriptorSet(fdsBytes); const message = set.messages.get(MapsMessage.typeName); const got = message?.fields .find((f) => f.name === "int32_msg_field") @@ -492,6 +515,7 @@ describe("DescriptorSet", () => { expect(got).toBe("map int32_msg_field = 10"); }); test("for enum value", () => { + const set = createDescriptorSet(fdsBytes); const e = set.enums.get(proto3.getEnumType(SimpleEnum).typeName); const got = e?.values .find((v) => v.name === "SIMPLE_ZERO") @@ -501,6 +525,7 @@ describe("DescriptorSet", () => { }); describe("getComments()", () => { describe("for file", () => { + const set = createDescriptorSet(fdsBytes); const file = set.files.find((file) => file.messages.some( (message) => message.typeName === MessageWithComments.typeName, @@ -528,6 +553,7 @@ describe("DescriptorSet", () => { }); }); test("for message", () => { + const set = createDescriptorSet(fdsBytes); const message = set.messages.get(MessageWithComments.typeName); const comments = message?.getComments(); expect(comments).toBeDefined(); @@ -541,6 +567,7 @@ describe("DescriptorSet", () => { } }); test("for field", () => { + const set = createDescriptorSet(fdsBytes); const field = set.messages .get(MessageWithComments.typeName) ?.fields.find((field) => field.name === "foo"); diff --git a/packages/protobuf/src/create-descriptor-set.ts b/packages/protobuf/src/create-descriptor-set.ts index a85ebb515..9de484465 100644 --- a/packages/protobuf/src/create-descriptor-set.ts +++ b/packages/protobuf/src/create-descriptor-set.ts @@ -68,7 +68,8 @@ export function createDescriptorSet( input: FileDescriptorProto[] | FileDescriptorSet | Uint8Array, options?: CreateDescriptorSetOptions, ): DescriptorSet { - const cart = { + const cart: Cart = { + files: [], enums: new Map(), messages: new Map(), services: new Map(), @@ -82,7 +83,7 @@ export function createDescriptorSet( ? FileDescriptorSet.fromBinary(input).file : input; const resolverByEdition = new Map(); - const files = fileDescriptors.map((proto) => { + for (const proto of fileDescriptors) { const edition = proto.edition ?? parseFileSyntax(proto.syntax, proto.edition).edition; let resolveFeatures = resolverByEdition.get(edition); @@ -94,9 +95,9 @@ export function createDescriptorSet( ); resolverByEdition.set(edition, resolveFeatures); } - return newFile(proto, cart, resolveFeatures); - }); - return { files, ...cart }; + addFile(proto, cart, resolveFeatures); + } + return cart; } /** @@ -129,6 +130,7 @@ interface CreateDescriptorSetOptions { * use to resolve reference when creating descriptors. */ interface Cart { + files: DescFile[]; enums: Map; messages: Map; services: Map; @@ -139,11 +141,11 @@ interface Cart { /** * Create a descriptor for a file. */ -function newFile( +function addFile( proto: FileDescriptorProto, cart: Cart, resolveFeatures: FeatureResolverFn, -): DescFile { +): void { assert(proto.name, `invalid FileDescriptorProto: missing name`); const file: DescFile = { kind: "file", @@ -151,6 +153,7 @@ function newFile( deprecated: proto.options?.deprecated ?? false, ...parseFileSyntax(proto.syntax, proto.edition), name: proto.name.replace(/\.proto/, ""), + dependencies: findFileDependencies(proto, cart), enums: [], messages: [], extensions: [], @@ -192,7 +195,7 @@ function newFile( addExtensions(message, cart, resolveFeatures); } cart.mapEntries.clear(); // map entries are local to the file, we can safely discard - return file; + cart.files.push(file); } /** @@ -806,6 +809,20 @@ function parseFileSyntax( }; } +/** + * Resolve dependencies of FileDescriptorProto to DescFile. + */ +function findFileDependencies( + proto: FileDescriptorProto, + cart: Cart, +): DescFile[] { + return proto.dependency.map((wantName) => { + const dep = cart.files.find((f) => f.proto.name === wantName); + assert(dep); + return dep; + }); +} + /** * Create a fully qualified name for a protobuf type or extension field. * diff --git a/packages/protobuf/src/descriptor-set.ts b/packages/protobuf/src/descriptor-set.ts index fb291dfcf..415866e14 100644 --- a/packages/protobuf/src/descriptor-set.ts +++ b/packages/protobuf/src/descriptor-set.ts @@ -83,7 +83,7 @@ export type AnyDesc = * Describes a protobuf source file. */ export interface DescFile { - kind: "file"; + readonly kind: "file"; /** * The syntax specified in the protobuf source. */ @@ -105,6 +105,10 @@ export interface DescFile { * For a protobuf file `foo/bar.proto`, this is `foo/bar`. */ readonly name: string; + /** + * Files imported by this file. + */ + readonly dependencies: DescFile[]; /** * Top-level enumerations declared in this file. * Note that more enumerations might be declared within message declarations. @@ -155,7 +159,7 @@ export interface DescFile { * Describes an enumeration in a protobuf source file. */ export interface DescEnum { - kind: "enum"; + readonly kind: "enum"; /** * The fully qualified name of the enumeration. (We omit the leading dot.) */ @@ -252,7 +256,7 @@ export interface DescEnumValue { * Describes a message declaration in a protobuf source file. */ export interface DescMessage { - kind: "message"; + readonly kind: "message"; /** * The fully qualified name of the message. (We omit the leading dot.) */ @@ -324,7 +328,7 @@ export interface DescMessage { */ export type DescField = DescFieldCommon & (DescFieldScalar | DescFieldMessage | DescFieldEnum | DescFieldMap) & { - kind: "field"; + readonly kind: "field"; /** * The message this field is declared on. @@ -337,7 +341,7 @@ export type DescField = DescFieldCommon & */ export type DescExtension = DescFieldCommon & (DescFieldScalar | DescFieldMessage | DescFieldEnum | DescFieldMap) & { - kind: "extension"; + readonly kind: "extension"; /** * The fully qualified name of the extension. @@ -637,7 +641,7 @@ interface DescFieldMapValueScalar { * Describes a oneof group in a protobuf source file. */ export interface DescOneof { - kind: "oneof"; + readonly kind: "oneof"; /** * The name of the oneof group, as specified in the protobuf source. */ @@ -678,7 +682,7 @@ export interface DescOneof { * Describes a service declaration in a protobuf source file. */ export interface DescService { - kind: "service"; + readonly kind: "service"; /** * The fully qualified name of the service. (We omit the leading dot.) */ @@ -721,7 +725,7 @@ export interface DescService { * Describes an RPC declaration in a protobuf source file. */ export interface DescMethod { - kind: "rpc"; + readonly kind: "rpc"; /** * The name of the RPC, as specified in the protobuf source. */