Skip to content

Commit

Permalink
Expose experimental edition features in descriptor sets (#627)
Browse files Browse the repository at this point in the history
  • Loading branch information
@timostamm
timostamm authored Nov 28, 2023
1 parent 5f95220 commit ae8a3ae
Show file tree
Hide file tree
Showing 8 changed files with 569 additions and 78 deletions.
210 changes: 202 additions & 8 deletions packages/protobuf-test/src/descriptor-set.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,17 @@

import { describe, expect, test } from "@jest/globals";
import { readFileSync } from "fs";
import type { AnyDesc } from "@bufbuild/protobuf";
import {
createDescriptorSet,
Edition,
FeatureSet,
FeatureSet_EnumType,
FeatureSet_JsonFormat,
FeatureSet_MessageEncoding,
FeatureSet_RepeatedFieldEncoding,
FeatureSet_Utf8Validation,
FeatureSet_FieldPresence,
proto3,
ScalarType,
} from "@bufbuild/protobuf";
Expand All @@ -40,12 +48,32 @@ describe("DescriptorSet", () => {
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("proto2");
expect(descFile?.edition).toBe(Edition.EDITION_PROTO2);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.EXPLICIT,
enumType: FeatureSet_EnumType.CLOSED,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.EXPANDED,
utf8Validation: FeatureSet_Utf8Validation.NONE,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
}),
);
});
test("proto3 syntax", () => {
const descFile = set.files.find((f) => f.name == "extra/proto3");
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("proto3");
expect(descFile?.edition).toBe(Edition.EDITION_PROTO3);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.IMPLICIT,
enumType: FeatureSet_EnumType.OPEN,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED,
utf8Validation: FeatureSet_Utf8Validation.VERIFY,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.ALLOW,
}),
);
});
test("edition 2023", () => {
const descFile = set.files.find(
Expand All @@ -54,6 +82,132 @@ describe("DescriptorSet", () => {
expect(descFile).toBeDefined();
expect(descFile?.syntax).toBe("editions");
expect(descFile?.edition).toBe(Edition.EDITION_2023);
expect(descFile?.getFeatures()).toStrictEqual(
new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.EXPLICIT,
enumType: FeatureSet_EnumType.OPEN,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.PACKED,
utf8Validation: FeatureSet_Utf8Validation.VERIFY,
messageEncoding: FeatureSet_MessageEncoding.LENGTH_PREFIXED,
jsonFormat: FeatureSet_JsonFormat.ALLOW,
}),
);
});
describe("edition feature options", () => {
test("file options should apply to all elements", async () => {
const bin = await new UpstreamProtobuf().compileToDescriptorSet(`
edition = "2023";
option features.field_presence = IMPLICIT;
option features.enum_type = CLOSED;
option features.repeated_field_encoding = EXPANDED;
option features.utf8_validation = NONE;
option features.message_encoding = DELIMITED;
option features.json_format = LEGACY_BEST_EFFORT;
message M {
message NestedMessage {
message DeepNestedMessage {}
enum DeepNestedEnum { A = 0; }
}
enum NestedEnum { A = 0; }
}
enum E { A = 0; }
service S {
rpc A(M) returns (M);
}`);
const { files, messages, enums, services } = createDescriptorSet(bin);
const wantFeatures = new FeatureSet({
fieldPresence: FeatureSet_FieldPresence.IMPLICIT,
enumType: FeatureSet_EnumType.CLOSED,
repeatedFieldEncoding: FeatureSet_RepeatedFieldEncoding.EXPANDED,
utf8Validation: FeatureSet_Utf8Validation.NONE,
messageEncoding: FeatureSet_MessageEncoding.DELIMITED,
jsonFormat: FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
});
expect(files[0].getFeatures()).toStrictEqual(wantFeatures);
expect(messages.get("M")?.getFeatures()).toStrictEqual(wantFeatures);
expect(messages.get("M.NestedMessage")?.getFeatures()).toStrictEqual(
wantFeatures,
);
expect(
messages.get("M.NestedMessage.DeepNestedMessage")?.getFeatures(),
).toStrictEqual(wantFeatures);
expect(enums.get("E")?.getFeatures()).toStrictEqual(wantFeatures);
expect(enums.get("M.NestedEnum")?.getFeatures()).toStrictEqual(
wantFeatures,
);
expect(
enums.get("M.NestedMessage.DeepNestedEnum")?.getFeatures(),
).toStrictEqual(wantFeatures);
expect(services.get("S")?.getFeatures()).toStrictEqual(wantFeatures);
expect(services.get("S")?.methods[0].getFeatures()).toStrictEqual(
wantFeatures,
);
});
test("message options should apply to nested elements", async () => {
const bin = await new UpstreamProtobuf().compileToDescriptorSet(`
edition="2023";
enum EnumJsonAllow { A = 0; }
message MessageJsonAllow {
int32 f = 1;
enum EnumJsonAllow { A = 0; }
message MessageJsonAllow {
int32 f = 1;
}
message MessageJsonLegacy {
option features.json_format = LEGACY_BEST_EFFORT;
int32 f = 1;
enum EnumJsonLegacy { A = 0; }
message MessageJsonLegacy {
int32 f = 1;
}
}
}`);
const { messages, enums } = createDescriptorSet(bin);
function expectJsonFormat(
jsonFormat: FeatureSet_JsonFormat,
desc: AnyDesc | undefined,
) {
expect(desc?.getFeatures().jsonFormat).toBe(jsonFormat);
if (desc?.kind == "message") {
for (const field of desc.fields) {
expect(field.getFeatures().jsonFormat).toBe(jsonFormat);
}
for (const oneof of desc.oneofs) {
expect(oneof.getFeatures().jsonFormat).toBe(jsonFormat);
}
}
if (desc?.kind == "enum") {
for (const value of desc.values) {
expect(value.getFeatures().jsonFormat).toBe(jsonFormat);
}
}
}
expectJsonFormat(FeatureSet_JsonFormat.ALLOW, enums.get("EnumJsonAllow"));
expectJsonFormat(
FeatureSet_JsonFormat.ALLOW,
messages.get("MessageJsonAllow"),
);
expectJsonFormat(
FeatureSet_JsonFormat.ALLOW,
enums.get("MessageJsonAllow.EnumJsonAllow"),
);
expectJsonFormat(
FeatureSet_JsonFormat.ALLOW,
messages.get("MessageJsonAllow.MessageJsonAllow"),
);
expectJsonFormat(
FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
messages.get("MessageJsonAllow.MessageJsonLegacy"),
);
expectJsonFormat(
FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
enums.get("MessageJsonAllow.MessageJsonLegacy.EnumJsonLegacy"),
);
expectJsonFormat(
FeatureSet_JsonFormat.LEGACY_BEST_EFFORT,
messages.get("MessageJsonAllow.MessageJsonLegacy.MessageJsonLegacy"),
);
});
});
describe("repeated field packing", () => {
test("proto2 is unpacked by default", async () => {
Expand Down Expand Up @@ -98,6 +252,47 @@ describe("DescriptorSet", () => {
expect(fields?.[3].packed).toBe(true);
expect(fields?.[4].packed).toBe(false);
});
test("edition2023 is packed by default", async () => {
const bin = await new UpstreamProtobuf().compileToDescriptorSet(`
edition="2023";
message M {
int32 not_packable = 1;
repeated bytes also_not_packable = 2;
repeated int32 default = 3;
repeated int32 explicitly_packed = 4 [features.repeated_field_encoding = PACKED];
repeated int32 explicitly_expanded = 5 [features.repeated_field_encoding = EXPANDED];
}
`);
const fields = createDescriptorSet(bin).messages.get("M")?.fields;
expect(fields?.[0].packedByDefault).toBe(true);
expect(fields?.[1].packedByDefault).toBe(false);
expect(fields?.[2].packedByDefault).toBe(true);
expect(fields?.[0].packed).toBe(true);
expect(fields?.[1].packed).toBe(false);
expect(fields?.[2].packed).toBe(true);
expect(fields?.[3].packed).toBe(true);
expect(fields?.[4].packed).toBe(false);
});
test("edition2023 with repeated_field_encoding file option", async () => {
const bin = await new UpstreamProtobuf().compileToDescriptorSet(`
edition="2023";
option features.repeated_field_encoding = EXPANDED;
message M {
int32 not_packable = 1;
repeated bytes also_not_packable = 2;
repeated int32 default = 3;
repeated int32 explicitly_packed = 4 [features.repeated_field_encoding = PACKED];
}
`);
const fields = createDescriptorSet(bin).messages.get("M")?.fields;
expect(fields?.[0].packedByDefault).toBe(true);
expect(fields?.[1].packedByDefault).toBe(false);
expect(fields?.[2].packedByDefault).toBe(true);
expect(fields?.[0].packed).toBe(false);
expect(fields?.[1].packed).toBe(false);
expect(fields?.[2].packed).toBe(false);
expect(fields?.[3].packed).toBe(true);
});
});
test("knows extension", () => {
const ext = set.extensions.get(
Expand Down Expand Up @@ -204,10 +399,8 @@ describe("DescriptorSet", () => {
}
});
});
});
describe("for message", () => {
const message = set.messages.get(MessageWithComments.typeName);
test("itself", () => {
test("for message", () => {
const message = set.messages.get(MessageWithComments.typeName);
const comments = message?.getComments();
expect(comments).toBeDefined();
if (comments) {
Expand All @@ -219,10 +412,11 @@ describe("DescriptorSet", () => {
expect(comments.trailing).toBeUndefined();
}
});
test("field", () => {
const comments = message?.fields
.find((field) => field.name === "foo")
?.getComments();
test("for field", () => {
const field = set.messages
.get(MessageWithComments.typeName)
?.fields.find((field) => field.name === "foo");
const comments = field?.getComments();
expect(comments).toBeDefined();
if (comments) {
expect(comments.leadingDetached).toStrictEqual([
Expand Down
2 changes: 1 addition & 1 deletion packages/protobuf/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"build:cjs": "../../node_modules/typescript/bin/tsc --project tsconfig.json --module commonjs --outDir ./dist/cjs --declaration --declarationDir ./dist/cjs && echo >./dist/cjs/package.json '{\"type\":\"commonjs\"}'",
"build:esm": "../../node_modules/typescript/bin/tsc --project tsconfig.json --module ES2015 --verbatimModuleSyntax --outDir ./dist/esm --declaration --declarationDir ./dist/esm",
"build:proxy": "node ../../scripts/gen-esm-proxy.mjs .",
"bootstrap:featureset-defaults": "upstream-inject-feature-defaults src/private/feature-set-defaults.ts",
"bootstrap:featureset-defaults": "upstream-inject-feature-defaults src/private/feature-set.ts",
"prebootstrap:wkt": "rm -rf .tmp && mkdir -p .tmp/google/protobuf && cp -rp src/google/protobuf/* .tmp/google/protobuf",
"bootstrap:wkt": "protoc --es_out=src --es_opt=bootstrap_wkt=true,ts_nocheck=false,target=ts --proto_path $(upstream-include wkt) $(upstream-files wkt) && license-header src/google/protobuf",
"postbootstrap:wkt": "diff >/dev/null -r .tmp/google/protobuf src/google/protobuf && cp -rp .tmp/google/protobuf/* src/google/protobuf || true",
Expand Down
Loading

0 comments on commit ae8a3ae

Please sign in to comment.