Skip to content

Commit

Permalink
Add "dependencies" to DescFile (#727)
Browse files Browse the repository at this point in the history
  • Loading branch information
@timostamm
timostamm authored Feb 26, 2024
1 parent 48d4ddc commit f3b5a6d
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 66 deletions.
127 changes: 77 additions & 50 deletions packages/protobuf-test/src/descriptor-set.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,56 +50,74 @@ import { join } from "node:path";
const fdsBytes = readFileSync("./descriptorset.binpb");

describe("DescriptorSet", () => {
const set = createDescriptorSet(fdsBytes);
test("proto2 syntax", () => {
const descFile = set.files.find((f) => f.name == "extra/proto2");
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("proto2");
expect(descFile?.edition).toBe(Edition.EDITION_PROTO2);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.EXPLICIT,
enumType: FeatureSet_EnumType.CLOSED,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.EXPANDED,
utf8Validation: FeatureSet_Utf8Validation.NONE,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
}),
);
});
test("proto3 syntax", () => {
const descFile = set.files.find((f) => f.name == "extra/proto3");
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("proto3");
expect(descFile?.edition).toBe(Edition.EDITION_PROTO3);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.IMPLICIT,
enumType: FeatureSet_EnumType.OPEN,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED,
utf8Validation: FeatureSet_Utf8Validation.VERIFY,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.ALLOW,
}),
);
});
test("edition 2023", () => {
const descFile = set.files.find(
(f) => f.name == "editions/edition2023-default-features",
);
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("editions");
expect(descFile?.edition).toBe(Edition.EDITION_2023);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.EXPLICIT,
enumType: FeatureSet_EnumType.OPEN,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED,
utf8Validation: FeatureSet_Utf8Validation.VERIFY,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.ALLOW,
}),
);
describe("file", () => {
test("proto2 syntax", () => {
const set = createDescriptorSet(fdsBytes);
const descFile = set.files.find((f) => f.name == "extra/proto2");
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("proto2");
expect(descFile?.edition).toBe(Edition.EDITION_PROTO2);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.EXPLICIT,
enumType: FeatureSet_EnumType.CLOSED,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.EXPANDED,
utf8Validation: FeatureSet_Utf8Validation.NONE,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
}),
);
});
test("proto3 syntax", () => {
const set = createDescriptorSet(fdsBytes);
const descFile = set.files.find((f) => f.name == "extra/proto3");
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("proto3");
expect(descFile?.edition).toBe(Edition.EDITION_PROTO3);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.IMPLICIT,
enumType: FeatureSet_EnumType.OPEN,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED,
utf8Validation: FeatureSet_Utf8Validation.VERIFY,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.ALLOW,
}),
);
});
test("edition 2023", () => {
const set = createDescriptorSet(fdsBytes);
const descFile = set.files.find(
(f) => f.name == "editions/edition2023-default-features",
);
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("editions");
expect(descFile?.edition).toBe(Edition.EDITION_2023);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.EXPLICIT,
enumType: FeatureSet_EnumType.OPEN,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED,
utf8Validation: FeatureSet_Utf8Validation.VERIFY,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.ALLOW,
}),
);
});
test("dependencies", async () => {
const fdsBin = await new UpstreamProtobuf().compileToDescriptorSet({
"a.proto": `syntax="proto3";
import "b.proto";
import "c.proto";`,
"b.proto": `syntax="proto3";`,
"c.proto": `syntax="proto3";`,
});
const set = createDescriptorSet(fdsBin);
const a = set.files[2];
expect(a.name).toBe("a");
expect(a.dependencies.length).toBe(2);
expect(a.dependencies.map((f) => f.name)).toStrictEqual(["b", "c"]);
});
});
describe("edition feature options", () => {
test("file options should apply to all elements", async () => {
Expand Down Expand Up @@ -423,6 +441,7 @@ describe("DescriptorSet", () => {
});
});
test("knows extension", () => {
const set = createDescriptorSet(fdsBytes);
const ext = set.extensions.get(
"protobuf_unittest.optional_int32_extension",
);
Expand All @@ -442,6 +461,7 @@ describe("DescriptorSet", () => {
);
});
test("knows nested extension", () => {
const set = createDescriptorSet(fdsBytes);
const ext = set.extensions.get(
"protobuf_unittest.TestNestedExtension.nested_string_extension",
);
Expand All @@ -465,6 +485,7 @@ describe("DescriptorSet", () => {
});
describe("declarationString()", () => {
test("for field with options", () => {
const set = createDescriptorSet(fdsBytes);
const message = set.messages.get(JsonNamesMessage.typeName);
expect(message).toBeDefined();
if (message !== undefined) {
Expand All @@ -475,6 +496,7 @@ describe("DescriptorSet", () => {
}
});
test("for field with labels", () => {
const set = createDescriptorSet(fdsBytes);
const message = set.messages.get(RepeatedScalarValuesMessage.typeName);
expect(message).toBeDefined();
if (message !== undefined) {
Expand All @@ -485,13 +507,15 @@ describe("DescriptorSet", () => {
}
});
test("for map field", () => {
const set = createDescriptorSet(fdsBytes);
const message = set.messages.get(MapsMessage.typeName);
const got = message?.fields
.find((f) => f.name === "int32_msg_field")
?.declarationString();
expect(got).toBe("map<int32, spec.MapsMessage> int32_msg_field = 10");
});
test("for enum value", () => {
const set = createDescriptorSet(fdsBytes);
const e = set.enums.get(proto3.getEnumType(SimpleEnum).typeName);
const got = e?.values
.find((v) => v.name === "SIMPLE_ZERO")
Expand All @@ -501,6 +525,7 @@ describe("DescriptorSet", () => {
});
describe("getComments()", () => {
describe("for file", () => {
const set = createDescriptorSet(fdsBytes);
const file = set.files.find((file) =>
file.messages.some(
(message) => message.typeName === MessageWithComments.typeName,
Expand Down Expand Up @@ -528,6 +553,7 @@ describe("DescriptorSet", () => {
});
});
test("for message", () => {
const set = createDescriptorSet(fdsBytes);
const message = set.messages.get(MessageWithComments.typeName);
const comments = message?.getComments();
expect(comments).toBeDefined();
Expand All @@ -541,6 +567,7 @@ describe("DescriptorSet", () => {
}
});
test("for field", () => {
const set = createDescriptorSet(fdsBytes);
const field = set.messages
.get(MessageWithComments.typeName)
?.fields.find((field) => field.name === "foo");
Expand Down
33 changes: 25 additions & 8 deletions packages/protobuf/src/create-descriptor-set.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ export function createDescriptorSet(
input: FileDescriptorProto[] | FileDescriptorSet | Uint8Array,
options?: CreateDescriptorSetOptions,
): DescriptorSet {
const cart = {
const cart: Cart = {
files: [],
enums: new Map<string, DescEnum>(),
messages: new Map<string, DescMessage>(),
services: new Map<string, DescService>(),
Expand All @@ -82,7 +83,7 @@ export function createDescriptorSet(
? FileDescriptorSet.fromBinary(input).file
: input;
const resolverByEdition = new Map<Edition, FeatureResolverFn>();
const files = fileDescriptors.map((proto) => {
for (const proto of fileDescriptors) {
const edition =
proto.edition ?? parseFileSyntax(proto.syntax, proto.edition).edition;
let resolveFeatures = resolverByEdition.get(edition);
Expand All @@ -94,9 +95,9 @@ export function createDescriptorSet(
);
resolverByEdition.set(edition, resolveFeatures);
}
return newFile(proto, cart, resolveFeatures);
});
return { files, ...cart };
addFile(proto, cart, resolveFeatures);
}
return cart;
}

/**
Expand Down Expand Up @@ -129,6 +130,7 @@ interface CreateDescriptorSetOptions {
* use to resolve reference when creating descriptors.
*/
interface Cart {
files: DescFile[];
enums: Map<string, DescEnum>;
messages: Map<string, DescMessage>;
services: Map<string, DescService>;
Expand All @@ -139,18 +141,19 @@ interface Cart {
/**
* Create a descriptor for a file.
*/
function newFile(
function addFile(
proto: FileDescriptorProto,
cart: Cart,
resolveFeatures: FeatureResolverFn,
): DescFile {
): void {
assert(proto.name, `invalid FileDescriptorProto: missing name`);
const file: DescFile = {
kind: "file",
proto,
deprecated: proto.options?.deprecated ?? false,
...parseFileSyntax(proto.syntax, proto.edition),
name: proto.name.replace(/\.proto/, ""),
dependencies: findFileDependencies(proto, cart),
enums: [],
messages: [],
extensions: [],
Expand Down Expand Up @@ -192,7 +195,7 @@ function newFile(
addExtensions(message, cart, resolveFeatures);
}
cart.mapEntries.clear(); // map entries are local to the file, we can safely discard
return file;
cart.files.push(file);
}

/**
Expand Down Expand Up @@ -806,6 +809,20 @@ function parseFileSyntax(
};
}

/**
* Resolve dependencies of FileDescriptorProto to DescFile.
*/
function findFileDependencies(
proto: FileDescriptorProto,
cart: Cart,
): DescFile[] {
return proto.dependency.map((wantName) => {
const dep = cart.files.find((f) => f.proto.name === wantName);
assert(dep);
return dep;
});
}

/**
* Create a fully qualified name for a protobuf type or extension field.
*
Expand Down
20 changes: 12 additions & 8 deletions packages/protobuf/src/descriptor-set.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ export type AnyDesc =
* Describes a protobuf source file.
*/
export interface DescFile {
kind: "file";
readonly kind: "file";
/**
* The syntax specified in the protobuf source.
*/
Expand All @@ -105,6 +105,10 @@ export interface DescFile {
* For a protobuf file `foo/bar.proto`, this is `foo/bar`.
*/
readonly name: string;
/**
* Files imported by this file.
*/
readonly dependencies: DescFile[];
/**
* Top-level enumerations declared in this file.
* Note that more enumerations might be declared within message declarations.
Expand Down Expand Up @@ -155,7 +159,7 @@ export interface DescFile {
* Describes an enumeration in a protobuf source file.
*/
export interface DescEnum {
kind: "enum";
readonly kind: "enum";
/**
* The fully qualified name of the enumeration. (We omit the leading dot.)
*/
Expand Down Expand Up @@ -252,7 +256,7 @@ export interface DescEnumValue {
* Describes a message declaration in a protobuf source file.
*/
export interface DescMessage {
kind: "message";
readonly kind: "message";
/**
* The fully qualified name of the message. (We omit the leading dot.)
*/
Expand Down Expand Up @@ -324,7 +328,7 @@ export interface DescMessage {
*/
export type DescField = DescFieldCommon &
(DescFieldScalar | DescFieldMessage | DescFieldEnum | DescFieldMap) & {
kind: "field";
readonly kind: "field";

/**
* The message this field is declared on.
Expand All @@ -337,7 +341,7 @@ export type DescField = DescFieldCommon &
*/
export type DescExtension = DescFieldCommon &
(DescFieldScalar | DescFieldMessage | DescFieldEnum | DescFieldMap) & {
kind: "extension";
readonly kind: "extension";

/**
* The fully qualified name of the extension.
Expand Down Expand Up @@ -637,7 +641,7 @@ interface DescFieldMapValueScalar {
* Describes a oneof group in a protobuf source file.
*/
export interface DescOneof {
kind: "oneof";
readonly kind: "oneof";
/**
* The name of the oneof group, as specified in the protobuf source.
*/
Expand Down Expand Up @@ -678,7 +682,7 @@ export interface DescOneof {
* Describes a service declaration in a protobuf source file.
*/
export interface DescService {
kind: "service";
readonly kind: "service";
/**
* The fully qualified name of the service. (We omit the leading dot.)
*/
Expand Down Expand Up @@ -721,7 +725,7 @@ export interface DescService {
* Describes an RPC declaration in a protobuf source file.
*/
export interface DescMethod {
kind: "rpc";
readonly kind: "rpc";
/**
* The name of the RPC, as specified in the protobuf source.
*/
Expand Down

0 comments on commit f3b5a6d

Please sign in to comment.