From e16130a36d4e00b8c6e24997ff2085760482ce44 Mon Sep 17 00:00:00 2001 From: Timo Stamm Date: Thu, 16 Nov 2023 18:54:36 +0100 Subject: [PATCH] Make edition feature defaults from `protoc` available --- package-lock.json | 3 +- packages/protobuf-test/src/json-names.test.ts | 153 ++++++------------ packages/upstream-protobuf/README.md | 5 +- .../bin/conformance_test_runner.mjs | 2 +- packages/upstream-protobuf/bin/protoc.mjs | 2 +- .../upstream-protobuf/bin/upstream-files.mjs | 2 +- .../bin/upstream-include.mjs | 2 +- .../bin/upstream-inject-feature-defaults.mjs | 95 +++++++++++ .../upstream-protobuf/bin/upstream-warmup.mjs | 2 +- packages/upstream-protobuf/index.d.ts | 26 +++ .../upstream-protobuf/{lib.mjs => index.mjs} | 80 ++++++++- packages/upstream-protobuf/package.json | 8 +- 12 files changed, 261 insertions(+), 119 deletions(-) create mode 100755 packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs create mode 100644 packages/upstream-protobuf/index.d.ts rename packages/upstream-protobuf/{lib.mjs => index.mjs} (86%) diff --git a/package-lock.json b/package-lock.json index 935a84f9a..c0adf719a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "protobuf-es-6", + "name": "protobuf-es", "lockfileVersion": 3, "requires": true, "packages": { @@ -6443,6 +6443,7 @@ "protoc": "bin/protoc.mjs", "upstream-files": "bin/upstream-files.mjs", "upstream-include": "bin/upstream-include.mjs", + "upstream-inject-feature-defaults": "bin/upstream-inject-feature-defaults.mjs", "upstream-warmup": "bin/upstream-warmup.mjs" } }, diff --git a/packages/protobuf-test/src/json-names.test.ts b/packages/protobuf-test/src/json-names.test.ts index 3f743b276..b52fb1394 100644 --- a/packages/protobuf-test/src/json-names.test.ts +++ b/packages/protobuf-test/src/json-names.test.ts @@ -13,115 +13,58 @@ // limitations under the License. import { expect, test } from "@jest/globals"; -import { mkdtempSync, readFileSync, writeFileSync } from "fs"; -import { join } from "path"; -import { tmpdir } from "os"; -import { spawnSync } from "child_process"; import { FileDescriptorSet, proto3, ScalarType } from "@bufbuild/protobuf"; +import { UpstreamProtobuf } from "upstream-protobuf"; -test("JSON names equal protoc", () => { - expect(getProtocJsonName("foo_bar")).toBe("fooBar"); - expectRuntimeJsonNameEqualsProtocJsonName("foo_bar"); - expectRuntimeJsonNameEqualsProtocJsonName("__proto__"); - expectRuntimeJsonNameEqualsProtocJsonName("fieldname1"); - expectRuntimeJsonNameEqualsProtocJsonName("field_name2"); - expectRuntimeJsonNameEqualsProtocJsonName("_field_name3"); - expectRuntimeJsonNameEqualsProtocJsonName("field__name4_"); - expectRuntimeJsonNameEqualsProtocJsonName("field0name5"); - expectRuntimeJsonNameEqualsProtocJsonName("field_0_name6"); - expectRuntimeJsonNameEqualsProtocJsonName("fieldName7"); - expectRuntimeJsonNameEqualsProtocJsonName("FieldName8"); - expectRuntimeJsonNameEqualsProtocJsonName("field_Name9"); - expectRuntimeJsonNameEqualsProtocJsonName("Field_Name10"); - expectRuntimeJsonNameEqualsProtocJsonName("FIELD_NAME11"); - expectRuntimeJsonNameEqualsProtocJsonName("FIELD_name12"); - expectRuntimeJsonNameEqualsProtocJsonName("__field_name13"); - expectRuntimeJsonNameEqualsProtocJsonName("__Field_name14"); - expectRuntimeJsonNameEqualsProtocJsonName("field__name15"); - expectRuntimeJsonNameEqualsProtocJsonName("field__Name16"); - expectRuntimeJsonNameEqualsProtocJsonName("field_name17__"); - expectRuntimeJsonNameEqualsProtocJsonName("Field_name18__"); +test("JSON names equal protoc", async () => { + const names = [ + "foo_bar", + "__proto__", + "fieldname1", + "field_name2", + "_field_name3", + "field__name4_", + "field0name5", + "field_0_name6", + "fieldName7", + "FieldName8", + "field_Name9", + "Field_Name10", + "FIELD_NAME11", + "FIELD_name12", + "__field_name13", + "__Field_name14", + "field__name15", + "field__Name16", + "field_name17__", + "Field_name18__", + ]; + const protocNames = await getProtocJsonNames(names); + const runtimeNames = getRuntimeJsonNames(names); + expect(runtimeNames).toStrictEqual(protocNames); }); -// expectRuntimeJsonNameEqualsProtocJsonName takes the given proto field name -// and runs it through protoc to get the JSON name. -// The result is compared to our own implementation of the algorithm. -// It is important that the implementations in JS and Go match the protoc -// implementation. -export function expectRuntimeJsonNameEqualsProtocJsonName(protoName: string) { - const want = getProtocJsonName(protoName); - const got = getRuntimeJsonName(protoName); - if (want === false) { - return; - } - expect(want).toBe(got); -} - -function getRuntimeJsonName(name: string): string { - const mt = proto3.makeMessageType("Fuzz", [ - { no: 1, kind: "scalar", T: ScalarType.BOOL, name: name }, - ]); - const fi = mt.fields.find(1); - if (!fi) { - throw new Error(); - } - return fi.jsonName; +function getRuntimeJsonNames(protoFieldNames: string[]) { + const mt = proto3.makeMessageType( + "M", + protoFieldNames.map( + (n, i) => + ({ no: i + 1, kind: "scalar", T: ScalarType.INT32, name: n }) as const, + ), + ); + return mt.fields.list().map((f) => f.jsonName); } -// getProtocJsonName runs protoc to get the "json name" for a field -function getProtocJsonName(protoName: string): string | false { - if (protoName.trim() !== protoName) { - return false; - } - const tempDir = mkdtempSync(join(tmpdir(), "node-protoc-workdir-")); - const inFilename = join(tempDir, "i.proto"); - const outFilename = join(tempDir, "o"); - const inData = [ - `syntax = "proto3";`, - `message I {`, - ` int32 ${protoName} = 1;`, - `}`, - ].join("\n"); - writeFileSync(inFilename, inData, { encoding: "utf-8" }); - const result = spawnSync( - "protoc", - ["-I", tempDir, inFilename, "--descriptor_set_out", outFilename], - { - encoding: "utf-8", - }, - ); - if (result.stderr.length > 0) { - if (result.stderr.indexOf("Expected field name.") >= 0) { - return false; - } - if (result.stderr.indexOf("Expected field number.") >= 0) { - return false; - } - if (result.stderr.indexOf('Expected ";".') >= 0) { - return false; - } - if (result.stderr.indexOf("Missing field number.") >= 0) { - return false; - } - if ( - result.stderr.indexOf( - "Invalid control characters encountered in text.", - ) >= 0 - ) { - return false; - } - throw new Error(result.stderr); - } - if (result.error) { - throw result.error; - } - if (result.status !== 0) { - throw new Error("exit code " + String(result.status)); - } - const fds = FileDescriptorSet.fromBinary(readFileSync(outFilename)); - const jsonName = fds.file[0].messageType[0].field[0].jsonName; - if (jsonName === undefined) { - throw new Error("missing json name"); - } - return jsonName; +async function getProtocJsonNames(protoFieldNames: string[]) { + const upstream = new UpstreamProtobuf(); + const bytes = await upstream.compileToDescriptorSet({ + "i.proto": [ + `syntax="proto3";`, + `message M {`, + ...protoFieldNames.map((n, i) => `int32 ${n} = ${i + 1};`), + `}`, + ].join("\n"), + }); + const fds = FileDescriptorSet.fromBinary(bytes); + return fds.file[0].messageType[0].field.map((f) => f.jsonName); } diff --git a/packages/upstream-protobuf/README.md b/packages/upstream-protobuf/README.md index 4282fb408..cf5f98e5d 100644 --- a/packages/upstream-protobuf/README.md +++ b/packages/upstream-protobuf/README.md @@ -1,7 +1,8 @@ # Upstream protobuf -This package provides `protoc`, `conformance_test_runner`, and related proto -files via npm "binaries". +This package provides `protoc`, `conformance_test_runner`, related proto files, +and feature-set defaults for editions via npm "binaries", and via an exported +class. To update this project to use a new version, update the version number in version.txt and run `make`. diff --git a/packages/upstream-protobuf/bin/conformance_test_runner.mjs b/packages/upstream-protobuf/bin/conformance_test_runner.mjs index 5ce68df48..bf62e9c9e 100755 --- a/packages/upstream-protobuf/bin/conformance_test_runner.mjs +++ b/packages/upstream-protobuf/bin/conformance_test_runner.mjs @@ -16,7 +16,7 @@ import { execFileSync } from "node:child_process"; import { argv, exit, stderr } from "node:process"; -import { UpstreamProtobuf } from "../lib.mjs"; +import { UpstreamProtobuf } from "../index.mjs"; const upstream = new UpstreamProtobuf(); diff --git a/packages/upstream-protobuf/bin/protoc.mjs b/packages/upstream-protobuf/bin/protoc.mjs index 87f5cda4a..7124d6537 100755 --- a/packages/upstream-protobuf/bin/protoc.mjs +++ b/packages/upstream-protobuf/bin/protoc.mjs @@ -16,7 +16,7 @@ import { execFileSync } from "node:child_process"; import { argv, exit, stderr } from "node:process"; -import { UpstreamProtobuf } from "../lib.mjs"; +import { UpstreamProtobuf } from "../index.mjs"; const upstream = new UpstreamProtobuf(); diff --git a/packages/upstream-protobuf/bin/upstream-files.mjs b/packages/upstream-protobuf/bin/upstream-files.mjs index d020c8e60..5e95e9486 100755 --- a/packages/upstream-protobuf/bin/upstream-files.mjs +++ b/packages/upstream-protobuf/bin/upstream-files.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {argv, exit, stderr, stdout} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; void main(argv.slice(2)); diff --git a/packages/upstream-protobuf/bin/upstream-include.mjs b/packages/upstream-protobuf/bin/upstream-include.mjs index 3f4927a55..d6aa938a5 100755 --- a/packages/upstream-protobuf/bin/upstream-include.mjs +++ b/packages/upstream-protobuf/bin/upstream-include.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {argv, exit, stderr, stdout} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; void main(argv.slice(2)); diff --git a/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs b/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs new file mode 100755 index 000000000..d8651057e --- /dev/null +++ b/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +// Copyright 2021-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {argv, exit, stdout, stderr} from "node:process"; +import {parseArgs} from "node:util"; +import {readFileSync, writeFileSync} from "node:fs"; +import {UpstreamProtobuf} from "../index.mjs"; + +void main(argv.slice(2)); + +async function main(args) { + let min, max, positionals; + try { + ({values: {min, max}, positionals} = parseArgs({ + args, + options: { + min: { type: 'string' }, + max: { type: 'string' }, + }, + allowPositionals: true, + })); + } catch { + exitUsage(); + } + const upstream = new UpstreamProtobuf(); + const defaults = await upstream.getFeatureSetDefaults(min, max); + stdout.write(`Injecting google.protobuf.FeatureSetDefaults into ${positionals.length} files...\n`); + for (const path of positionals) { + const content = readFileSync(path, "utf-8"); + const r = inject(content, `"${defaults.toString("base64url")}"`); + if (!r.ok) { + stderr.write(`Error injecting into ${path}: ${r.message}\n`); + exit(1); + } + if (r.newContent === content) { + stdout.write(`- ${path} - no changes\n`); + continue; + } + writeFileSync(path, r.newContent); + stdout.write(`- ${path} - updated\n`); + } +} + +/** + * @typedef {object} InjectSuccess + * @property {true} ok + * @property {string} newContent + */ +/** + * @typedef {object} InjectError + * @property {false} ok + * @property {string} message + */ +/** + * @param {string} content + * @param {string} contentToInject + * @return {InjectSuccess | InjectError} + */ +function inject(content, contentToInject) { + const tokenStart = "/*upstream-inject-feature-defaults-start*/"; + const tokenEnd = "/*upstream-inject-feature-defaults-end*/"; + const indexStart = content.indexOf(tokenStart); + const indexEnd = content.indexOf(tokenEnd); + if (indexStart < 0 || indexEnd < 0) { + return {ok: false, message: "missing comment annotations"}; + } + if (indexEnd < indexStart) { + return {ok: false, message: "invalid comment annotations"}; + } + const head = content.substring(0, indexStart + tokenStart.length); + const foot = content.substring(indexEnd); + const newContent = head + contentToInject + foot; + return {ok: true, newContent}; +} + +/** + * @return never + */ +function exitUsage() { + stderr.write(`USAGE: upstream-inject-feature-defaults [--min ] [--max ] \n`); + exit(1); +} diff --git a/packages/upstream-protobuf/bin/upstream-warmup.mjs b/packages/upstream-protobuf/bin/upstream-warmup.mjs index 86a3b16a9..a52f38a9e 100755 --- a/packages/upstream-protobuf/bin/upstream-warmup.mjs +++ b/packages/upstream-protobuf/bin/upstream-warmup.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {exit, stderr} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; const upstream = new UpstreamProtobuf(); upstream.warmup().then(() => exit(0), reason => { diff --git a/packages/upstream-protobuf/index.d.ts b/packages/upstream-protobuf/index.d.ts new file mode 100644 index 000000000..0d0e71577 --- /dev/null +++ b/packages/upstream-protobuf/index.d.ts @@ -0,0 +1,26 @@ +// Copyright 2021-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export declare class UpstreamProtobuf { + constructor(temp?: string, version?: string); + + getProtocPath(): Promise; + + getFeatureSetDefaults( + minimumEdition?: string, + maximumEdition?: string, + ): Promise; + + compileToDescriptorSet(files: Record): Promise; +} diff --git a/packages/upstream-protobuf/lib.mjs b/packages/upstream-protobuf/index.mjs similarity index 86% rename from packages/upstream-protobuf/lib.mjs rename to packages/upstream-protobuf/index.mjs index 6690d503d..6b21f66e9 100644 --- a/packages/upstream-protobuf/lib.mjs +++ b/packages/upstream-protobuf/index.mjs @@ -13,15 +13,18 @@ // limitations under the License. import { + chmodSync, existsSync, - writeFileSync, - readFileSync, mkdirSync, - chmodSync, + mkdtempSync, readdirSync, + readFileSync, + writeFileSync, + rmSync, } from "node:fs"; +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { join as joinPath, dirname, relative as relativePath } from "node:path"; +import { dirname, join as joinPath, relative as relativePath } from "node:path"; import os from "node:os"; import { unzipSync } from "fflate"; import micromatch from "micromatch"; @@ -117,6 +120,71 @@ export class UpstreamProtobuf { ]); } + /** + * @param {Record} files + * @return {Promise} + */ + async compileToDescriptorSet(files) { + const protocPath = await this.getProtocPath(); + const tempDir = mkdtempSync( + joinPath(this.#temp, "compile-descriptor-set-"), + ); + try { + writeTree(Object.entries(files), tempDir); + const outPath = joinPath(tempDir, "desc.bin"); + execFileSync( + protocPath, + [ + "--descriptor_set_out", + outPath, + "--proto_path", + tempDir, + ...Object.keys(files), + ], + { + shell: false, + stdio: "ignore", + }, + ); + return readFileSync(outPath); + } finally { + rmSync(tempDir, { recursive: true }); + } + } + + /** + * @param {string} [minimumEdition] + * @param {string} [maximumEdition] + * @return Promise + */ + async getFeatureSetDefaults(minimumEdition, maximumEdition) { + const binPath = this.#getTempPath( + "feature-set-defaults", + `min-${minimumEdition ?? "default"}-max-${ + maximumEdition ?? "default" + }.bin`, + ); + if (!existsSync(binPath)) { + const protocPath = await this.getProtocPath(); + const args = [ + "--experimental_edition_defaults_out", + binPath, + "google/protobuf/descriptor.proto", + ]; + if (minimumEdition !== undefined) { + args.push("--experimental_edition_defaults_minimum", minimumEdition); + } + if (maximumEdition !== undefined) { + args.push("--experimental_edition_defaults_maximum", maximumEdition); + } + execFileSync(protocPath, args, { + shell: false, + stdio: "ignore", + }); + } + return readFileSync(binPath); + } + /** * @return {Promise} */ @@ -420,7 +488,7 @@ export class UpstreamProtobuf { } /** - * @param {Array<[string, Uint8Array]>} files + * @param {Array<[string, Uint8Array|string]>} files * @param {string} [dir] */ function writeTree(files, dir = ".") { @@ -439,6 +507,7 @@ function writeTree(files, dir = ".") { */ function lsfiles(dir) { const hits = []; + function ls(dir) { for (const ent of readdirSync(dir, { withFileTypes: true })) { const entPath = joinPath(dir, ent.name); @@ -449,6 +518,7 @@ function lsfiles(dir) { } } } + ls(dir); return hits.map((path) => relativePath(dir, path)); } diff --git a/packages/upstream-protobuf/package.json b/packages/upstream-protobuf/package.json index d4e527a39..66c3fb222 100644 --- a/packages/upstream-protobuf/package.json +++ b/packages/upstream-protobuf/package.json @@ -10,7 +10,13 @@ "conformance_test_runner": "bin/conformance_test_runner.mjs", "upstream-files": "bin/upstream-files.mjs", "upstream-include": "bin/upstream-include.mjs", - "upstream-warmup": "bin/upstream-warmup.mjs" + "upstream-warmup": "bin/upstream-warmup.mjs", + "upstream-inject-feature-defaults": "bin/upstream-inject-feature-defaults.mjs" + }, + "exports": { + ".": { + "import": "./index.mjs" + } }, "dependencies": { "fflate": "^0.8.1",