Skip to content

Commit

Permalink
Make edition feature defaults from protoc available
Browse files Browse the repository at this point in the history
  • Loading branch information
timostamm committed Nov 16, 2023
1 parent f7375ad commit e16130a
Show file tree
Hide file tree
Showing 12 changed files with 261 additions and 119 deletions.
3 changes: 2 additions & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

153 changes: 48 additions & 105 deletions packages/protobuf-test/src/json-names.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,115 +13,58 @@
// limitations under the License.

import { expect, test } from "@jest/globals";
import { mkdtempSync, readFileSync, writeFileSync } from "fs";
import { join } from "path";
import { tmpdir } from "os";
import { spawnSync } from "child_process";
import { FileDescriptorSet, proto3, ScalarType } from "@bufbuild/protobuf";
import { UpstreamProtobuf } from "upstream-protobuf";

test("JSON names equal protoc", () => {
expect(getProtocJsonName("foo_bar")).toBe("fooBar");
expectRuntimeJsonNameEqualsProtocJsonName("foo_bar");
expectRuntimeJsonNameEqualsProtocJsonName("__proto__");
expectRuntimeJsonNameEqualsProtocJsonName("fieldname1");
expectRuntimeJsonNameEqualsProtocJsonName("field_name2");
expectRuntimeJsonNameEqualsProtocJsonName("_field_name3");
expectRuntimeJsonNameEqualsProtocJsonName("field__name4_");
expectRuntimeJsonNameEqualsProtocJsonName("field0name5");
expectRuntimeJsonNameEqualsProtocJsonName("field_0_name6");
expectRuntimeJsonNameEqualsProtocJsonName("fieldName7");
expectRuntimeJsonNameEqualsProtocJsonName("FieldName8");
expectRuntimeJsonNameEqualsProtocJsonName("field_Name9");
expectRuntimeJsonNameEqualsProtocJsonName("Field_Name10");
expectRuntimeJsonNameEqualsProtocJsonName("FIELD_NAME11");
expectRuntimeJsonNameEqualsProtocJsonName("FIELD_name12");
expectRuntimeJsonNameEqualsProtocJsonName("__field_name13");
expectRuntimeJsonNameEqualsProtocJsonName("__Field_name14");
expectRuntimeJsonNameEqualsProtocJsonName("field__name15");
expectRuntimeJsonNameEqualsProtocJsonName("field__Name16");
expectRuntimeJsonNameEqualsProtocJsonName("field_name17__");
expectRuntimeJsonNameEqualsProtocJsonName("Field_name18__");
test("JSON names equal protoc", async () => {
const names = [
"foo_bar",
"__proto__",
"fieldname1",
"field_name2",
"_field_name3",
"field__name4_",
"field0name5",
"field_0_name6",
"fieldName7",
"FieldName8",
"field_Name9",
"Field_Name10",
"FIELD_NAME11",
"FIELD_name12",
"__field_name13",
"__Field_name14",
"field__name15",
"field__Name16",
"field_name17__",
"Field_name18__",
];
const protocNames = await getProtocJsonNames(names);
const runtimeNames = getRuntimeJsonNames(names);
expect(runtimeNames).toStrictEqual(protocNames);
});

// expectRuntimeJsonNameEqualsProtocJsonName takes the given proto field name
// and runs it through protoc to get the JSON name.
// The result is compared to our own implementation of the algorithm.
// It is important that the implementations in JS and Go match the protoc
// implementation.
export function expectRuntimeJsonNameEqualsProtocJsonName(protoName: string) {
const want = getProtocJsonName(protoName);
const got = getRuntimeJsonName(protoName);
if (want === false) {
return;
}
expect(want).toBe(got);
}

function getRuntimeJsonName(name: string): string {
const mt = proto3.makeMessageType("Fuzz", [
{ no: 1, kind: "scalar", T: ScalarType.BOOL, name: name },
]);
const fi = mt.fields.find(1);
if (!fi) {
throw new Error();
}
return fi.jsonName;
function getRuntimeJsonNames(protoFieldNames: string[]) {
const mt = proto3.makeMessageType(
"M",
protoFieldNames.map(
(n, i) =>
({ no: i + 1, kind: "scalar", T: ScalarType.INT32, name: n }) as const,
),
);
return mt.fields.list().map((f) => f.jsonName);
}

// getProtocJsonName runs protoc to get the "json name" for a field
function getProtocJsonName(protoName: string): string | false {
if (protoName.trim() !== protoName) {
return false;
}
const tempDir = mkdtempSync(join(tmpdir(), "node-protoc-workdir-"));
const inFilename = join(tempDir, "i.proto");
const outFilename = join(tempDir, "o");
const inData = [
`syntax = "proto3";`,
`message I {`,
` int32 ${protoName} = 1;`,
`}`,
].join("\n");
writeFileSync(inFilename, inData, { encoding: "utf-8" });
const result = spawnSync(
"protoc",
["-I", tempDir, inFilename, "--descriptor_set_out", outFilename],
{
encoding: "utf-8",
},
);
if (result.stderr.length > 0) {
if (result.stderr.indexOf("Expected field name.") >= 0) {
return false;
}
if (result.stderr.indexOf("Expected field number.") >= 0) {
return false;
}
if (result.stderr.indexOf('Expected ";".') >= 0) {
return false;
}
if (result.stderr.indexOf("Missing field number.") >= 0) {
return false;
}
if (
result.stderr.indexOf(
"Invalid control characters encountered in text.",
) >= 0
) {
return false;
}
throw new Error(result.stderr);
}
if (result.error) {
throw result.error;
}
if (result.status !== 0) {
throw new Error("exit code " + String(result.status));
}
const fds = FileDescriptorSet.fromBinary(readFileSync(outFilename));
const jsonName = fds.file[0].messageType[0].field[0].jsonName;
if (jsonName === undefined) {
throw new Error("missing json name");
}
return jsonName;
async function getProtocJsonNames(protoFieldNames: string[]) {
const upstream = new UpstreamProtobuf();
const bytes = await upstream.compileToDescriptorSet({
"i.proto": [
`syntax="proto3";`,
`message M {`,
...protoFieldNames.map((n, i) => `int32 ${n} = ${i + 1};`),
`}`,
].join("\n"),
});
const fds = FileDescriptorSet.fromBinary(bytes);
return fds.file[0].messageType[0].field.map((f) => f.jsonName);
}
5 changes: 3 additions & 2 deletions packages/upstream-protobuf/README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Upstream protobuf

This package provides `protoc`, `conformance_test_runner`, and related proto
files via npm "binaries".
This package provides `protoc`, `conformance_test_runner`, related proto files,
and feature-set defaults for editions via npm "binaries", and via an exported
class.

To update this project to use a new version, update the version number in
version.txt and run `make`.
2 changes: 1 addition & 1 deletion packages/upstream-protobuf/bin/conformance_test_runner.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import { execFileSync } from "node:child_process";
import { argv, exit, stderr } from "node:process";
import { UpstreamProtobuf } from "../lib.mjs";
import { UpstreamProtobuf } from "../index.mjs";

const upstream = new UpstreamProtobuf();

Expand Down
2 changes: 1 addition & 1 deletion packages/upstream-protobuf/bin/protoc.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import { execFileSync } from "node:child_process";
import { argv, exit, stderr } from "node:process";
import { UpstreamProtobuf } from "../lib.mjs";
import { UpstreamProtobuf } from "../index.mjs";

const upstream = new UpstreamProtobuf();

Expand Down
2 changes: 1 addition & 1 deletion packages/upstream-protobuf/bin/upstream-files.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// limitations under the License.

import {argv, exit, stderr, stdout} from "node:process";
import {UpstreamProtobuf} from "../lib.mjs";
import {UpstreamProtobuf} from "../index.mjs";

void main(argv.slice(2));

Expand Down
2 changes: 1 addition & 1 deletion packages/upstream-protobuf/bin/upstream-include.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// limitations under the License.

import {argv, exit, stderr, stdout} from "node:process";
import {UpstreamProtobuf} from "../lib.mjs";
import {UpstreamProtobuf} from "../index.mjs";

void main(argv.slice(2));

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
#!/usr/bin/env node

// Copyright 2021-2023 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import {argv, exit, stdout, stderr} from "node:process";
import {parseArgs} from "node:util";
import {readFileSync, writeFileSync} from "node:fs";
import {UpstreamProtobuf} from "../index.mjs";

void main(argv.slice(2));

async function main(args) {
let min, max, positionals;
try {
({values: {min, max}, positionals} = parseArgs({
args,
options: {
min: { type: 'string' },
max: { type: 'string' },
},
allowPositionals: true,
}));
} catch {
exitUsage();
}
const upstream = new UpstreamProtobuf();
const defaults = await upstream.getFeatureSetDefaults(min, max);
stdout.write(`Injecting google.protobuf.FeatureSetDefaults into ${positionals.length} files...\n`);
for (const path of positionals) {
const content = readFileSync(path, "utf-8");
const r = inject(content, `"${defaults.toString("base64url")}"`);
if (!r.ok) {
stderr.write(`Error injecting into ${path}: ${r.message}\n`);
exit(1);
}
if (r.newContent === content) {
stdout.write(`- ${path} - no changes\n`);
continue;
}
writeFileSync(path, r.newContent);
stdout.write(`- ${path} - updated\n`);
}
}

/**
* @typedef {object} InjectSuccess
* @property {true} ok
* @property {string} newContent
*/
/**
* @typedef {object} InjectError
* @property {false} ok
* @property {string} message
*/
/**
* @param {string} content
* @param {string} contentToInject
* @return {InjectSuccess | InjectError}
*/
function inject(content, contentToInject) {
const tokenStart = "/*upstream-inject-feature-defaults-start*/";
const tokenEnd = "/*upstream-inject-feature-defaults-end*/";
const indexStart = content.indexOf(tokenStart);
const indexEnd = content.indexOf(tokenEnd);
if (indexStart < 0 || indexEnd < 0) {
return {ok: false, message: "missing comment annotations"};
}
if (indexEnd < indexStart) {
return {ok: false, message: "invalid comment annotations"};
}
const head = content.substring(0, indexStart + tokenStart.length);
const foot = content.substring(indexEnd);
const newContent = head + contentToInject + foot;
return {ok: true, newContent};
}

/**
* @return never
*/
function exitUsage() {
stderr.write(`USAGE: upstream-inject-feature-defaults [--min <mininum supported edition>] [--max <maximum supported edition>] <file-to-inject-into>\n`);

This comment has been minimized.

Copy link
@smaye81

smaye81 Nov 16, 2023

Member

nit (typo): minimum supported edition...

exit(1);
}
2 changes: 1 addition & 1 deletion packages/upstream-protobuf/bin/upstream-warmup.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// limitations under the License.

import {exit, stderr} from "node:process";
import {UpstreamProtobuf} from "../lib.mjs";
import {UpstreamProtobuf} from "../index.mjs";

const upstream = new UpstreamProtobuf();
upstream.warmup().then(() => exit(0), reason => {
Expand Down
26 changes: 26 additions & 0 deletions packages/upstream-protobuf/index.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Copyright 2021-2023 Buf Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

export declare class UpstreamProtobuf {
constructor(temp?: string, version?: string);

getProtocPath(): Promise<string>;

getFeatureSetDefaults(
minimumEdition?: string,
maximumEdition?: string,
): Promise<Uint8Array>;

compileToDescriptorSet(files: Record<string, string>): Promise<Uint8Array>;
}
Loading

0 comments on commit e16130a

Please sign in to comment.