From 5136ac8a16427704e7e64fc1f44fab089249f1d4 Mon Sep 17 00:00:00 2001 From: Timo Stamm Date: Thu, 16 Nov 2023 18:29:11 +0100 Subject: [PATCH 1/2] Make edition feature defaults from `protoc` available --- .eslintrc.cjs | 117 +++++++------- Makefile | 2 +- package-lock.json | 120 +++++++------- package.json | 10 +- packages/protobuf-test/src/json-names.test.ts | 153 ++++++------------ packages/upstream-protobuf/README.md | 5 +- .../bin/conformance_test_runner.mjs | 2 +- packages/upstream-protobuf/bin/protoc.mjs | 2 +- .../upstream-protobuf/bin/upstream-files.mjs | 2 +- .../bin/upstream-include.mjs | 2 +- .../bin/upstream-inject-feature-defaults.mjs | 95 +++++++++++ .../upstream-protobuf/bin/upstream-warmup.mjs | 2 +- packages/upstream-protobuf/index.d.ts | 26 +++ .../upstream-protobuf/{lib.mjs => index.mjs} | 80 ++++++++- packages/upstream-protobuf/package.json | 8 +- 15 files changed, 380 insertions(+), 246 deletions(-) create mode 100755 packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs create mode 100644 packages/upstream-protobuf/index.d.ts rename packages/upstream-protobuf/{lib.mjs => index.mjs} (86%) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 9342c7ee3..0c9c6df80 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -const {readdirSync, existsSync} = require("fs"); -const {join} = require("path"); +const { readdirSync, existsSync } = require("fs"); +const { join } = require("path"); module.exports = { env: { @@ -27,79 +27,76 @@ module.exports = { "packages/*/.tmp/**", "node_modules/**", ], - plugins: ["@typescript-eslint", "node", "import"], + plugins: ["@typescript-eslint", "n", "import"], // Rules and settings that do not require a non-default parser - extends: [ - "eslint:recommended", - ], + extends: ["eslint:recommended"], rules: { "no-console": "error", "import/no-duplicates": "off", }, settings: {}, overrides: [ - ...readdirSync("packages", {withFileTypes: true}) - .filter(entry => entry.isDirectory()) - .map(entry => join("packages", entry.name)) - .filter(dir => existsSync(join(dir, "tsconfig.json"))) - .map(dir => { - return { - files: [join(dir, "src/**/*.ts")], - parser: "@typescript-eslint/parser", - parserOptions: { - project: "./tsconfig.json", - tsconfigRootDir: dir, - }, - settings: { - "import/resolver": { - "typescript": { - "project": "packages/*/tsconfig.json", - } - } - }, - extends: [ - "plugin:@typescript-eslint/recommended", - "plugin:@typescript-eslint/recommended-requiring-type-checking", - "plugin:import/recommended", - "plugin:import/typescript", - ], - rules: { - "@typescript-eslint/strict-boolean-expressions": "error", - "@typescript-eslint/no-unnecessary-condition": "error", - "@typescript-eslint/array-type": "off", // we use complex typings, where Array is actually more readable than T[] - "@typescript-eslint/switch-exhaustiveness-check": "error", - "@typescript-eslint/prefer-nullish-coalescing": "error", - "@typescript-eslint/no-unnecessary-boolean-literal-compare": "error", - "@typescript-eslint/no-invalid-void-type": "error", - "@typescript-eslint/no-base-to-string": "error", - "import/no-cycle": "error", - "import/no-duplicates": "off", + ...readdirSync("packages", { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => join("packages", entry.name)) + .filter((dir) => existsSync(join(dir, "tsconfig.json"))) + .map((dir) => { + return { + files: [join(dir, "src/**/*.ts")], + parser: "@typescript-eslint/parser", + parserOptions: { + project: "./tsconfig.json", + tsconfigRootDir: dir, + }, + settings: { + "import/resolver": { + typescript: { + project: "packages/*/tsconfig.json", + }, }, - }; - }), + }, + extends: [ + "plugin:@typescript-eslint/recommended", + "plugin:@typescript-eslint/recommended-requiring-type-checking", + "plugin:import/recommended", + "plugin:import/typescript", + ], + rules: { + "@typescript-eslint/strict-boolean-expressions": "error", + "@typescript-eslint/no-unnecessary-condition": "error", + "@typescript-eslint/array-type": "off", // we use complex typings, where Array is actually more readable than T[] + "@typescript-eslint/switch-exhaustiveness-check": "error", + "@typescript-eslint/prefer-nullish-coalescing": "error", + "@typescript-eslint/no-unnecessary-boolean-literal-compare": + "error", + "@typescript-eslint/no-invalid-void-type": "error", + "@typescript-eslint/no-base-to-string": "error", + "import/no-cycle": "error", + "import/no-duplicates": "off", + }, + }; + }), // For scripts and configurations, use Node.js rules { files: ["**/*.{js,mjs,cjs}"], parserOptions: { ecmaVersion: 13, // ES2022 - https://eslint.org/docs/latest/use/configure/language-options#specifying-environments }, - extends: ["eslint:recommended", "plugin:node/recommended"], + extends: ["eslint:recommended", "plugin:n/recommended"], rules: { - "node/shebang": "off", // this plugin only determines shebang necessary for files that are in a package.json "bin" field - "node/exports-style": ["error", "module.exports"], - "node/file-extension-in-import": ["error", "always"], - "node/prefer-global/buffer": ["error", "always"], - "node/prefer-global/console": ["error", "always"], - "node/prefer-global/process": ["error", "always"], - "node/prefer-global/url-search-params": ["error", "always"], - "node/prefer-global/url": ["error", "always"], - "node/prefer-promises/dns": "error", - "node/prefer-promises/fs": "error", - "no-process-exit": "off", - "node/no-unsupported-features/node-builtins": ["error", { - "version": ">=16.0.0", - "ignores": [] - }] + "n/shebang": "off", // this rule reports _any_ shebang outside of an npm binary as an error + "n/prefer-global/process": "off", + "n/no-process-exit": "off", + "n/exports-style": ["error", "module.exports"], + "n/file-extension-in-import": ["error", "always"], + "n/prefer-global/buffer": ["error", "always"], + "n/prefer-global/console": ["error", "always"], + "n/prefer-global/url-search-params": ["error", "always"], + "n/prefer-global/url": ["error", "always"], + "n/prefer-promises/dns": "error", + "n/prefer-promises/fs": "error", + "n/no-unsupported-features/node-builtins": "error", + "n/no-unsupported-features/es-syntax": "error", }, }, ], diff --git a/Makefile b/Makefile index b889d4c74..6808efbe8 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ lint: node_modules $(BUILD)/protobuf $(BUILD)/protobuf-test $(BUILD)/protobuf-co .PHONY: format format: node_modules ## Format all files, adding license headers - npx prettier --write '**/*.{json,js,jsx,ts,tsx,css,mjs}' --log-level error + npx prettier --write '**/*.{json,js,jsx,ts,tsx,css,mjs,cjs}' --log-level error npx license-header --ignore packages/protobuf/src/google/varint.ts .PHONY: bench diff --git a/package-lock.json b/package-lock.json index a040bbe8a..c0adf719a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "protobuf-es-4", + "name": "protobuf-es", "lockfileVersion": 3, "requires": true, "packages": { @@ -29,7 +29,7 @@ "eslint": "^8.50.0", "eslint-import-resolver-typescript": "^3.6.0", "eslint-plugin-import": "^2.28.0", - "eslint-plugin-node": "^11.1.0", + "eslint-plugin-n": "^16.3.1", "jest": "^29.7.0", "prettier": "^3.0.0", "typescript": "^5.2.2" @@ -2095,6 +2095,18 @@ "version": "1.1.2", "license": "MIT" }, + "node_modules/builtin-modules": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", + "integrity": "sha512-zhaCDicdLuWN5UbN5IMnFqNMhNfo919sH85y2/ea+5Yg9TsTkeZxpL+JLbp6cgYFS4sRLp3YV4S6yDuqVWHYOw==", + "dev": true, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/builtins": { "version": "5.0.1", "dev": true, @@ -2730,22 +2742,23 @@ "ms": "^2.1.1" } }, - "node_modules/eslint-plugin-es": { - "version": "3.0.1", + "node_modules/eslint-plugin-es-x": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-es-x/-/eslint-plugin-es-x-7.3.0.tgz", + "integrity": "sha512-W9zIs+k00I/I13+Bdkl/zG1MEO07G97XjUSQuH117w620SJ6bHtLUmoMvkGA2oYnI/gNdr+G7BONLyYnFaLLEQ==", "dev": true, - "license": "MIT", "dependencies": { - "eslint-utils": "^2.0.0", - "regexpp": "^3.0.0" + "@eslint-community/eslint-utils": "^4.1.2", + "@eslint-community/regexpp": "^4.6.0" }, "engines": { - "node": ">=8.10.0" + "node": "^14.18.0 || >=16.0.0" }, "funding": { - "url": "https://github.com/sponsors/mysticatea" + "url": "https://github.com/sponsors/ota-meshi" }, "peerDependencies": { - "eslint": ">=4.19.1" + "eslint": ">=8" } }, "node_modules/eslint-plugin-import": { @@ -2805,31 +2818,31 @@ "semver": "bin/semver.js" } }, - "node_modules/eslint-plugin-node": { - "version": "11.1.0", + "node_modules/eslint-plugin-n": { + "version": "16.3.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-n/-/eslint-plugin-n-16.3.1.tgz", + "integrity": "sha512-w46eDIkxQ2FaTHcey7G40eD+FhTXOdKudDXPUO2n9WNcslze/i/HT2qJ3GXjHngYSGDISIgPNhwGtgoix4zeOw==", "dev": true, - "license": "MIT", "dependencies": { - "eslint-plugin-es": "^3.0.0", - "eslint-utils": "^2.0.0", - "ignore": "^5.1.1", - "minimatch": "^3.0.4", - "resolve": "^1.10.1", - "semver": "^6.1.0" + "@eslint-community/eslint-utils": "^4.4.0", + "builtins": "^5.0.1", + "eslint-plugin-es-x": "^7.1.0", + "get-tsconfig": "^4.7.0", + "ignore": "^5.2.4", + "is-builtin-module": "^3.2.1", + "is-core-module": "^2.12.1", + "minimatch": "^3.1.2", + "resolve": "^1.22.2", + "semver": "^7.5.3" }, "engines": { - "node": ">=8.10.0" + "node": ">=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" }, "peerDependencies": { - "eslint": ">=5.16.0" - } - }, - "node_modules/eslint-plugin-node/node_modules/semver": { - "version": "6.3.1", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" + "eslint": ">=7.0.0" } }, "node_modules/eslint-scope": { @@ -2847,28 +2860,6 @@ "url": "https://opencollective.com/eslint" } }, - "node_modules/eslint-utils": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^1.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/mysticatea" - } - }, - "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "1.3.0", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=4" - } - }, "node_modules/eslint-visitor-keys": { "version": "3.4.3", "dev": true, @@ -3546,6 +3537,21 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-builtin-module": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", + "integrity": "sha512-BSLE3HnV2syZ0FK0iMA/yUGplUeMmNz4AW5fnTunbCIqZi4vG3WjJT9FHMy5D69xmAYBHXQhJdALdpwVxV501A==", + "dev": true, + "dependencies": { + "builtin-modules": "^3.3.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-callable": { "version": "1.2.7", "dev": true, @@ -5105,17 +5111,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/regexpp": { - "version": "3.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/mysticatea" - } - }, "node_modules/require-directory": { "version": "2.1.1", "dev": true, @@ -6448,6 +6443,7 @@ "protoc": "bin/protoc.mjs", "upstream-files": "bin/upstream-files.mjs", "upstream-include": "bin/upstream-include.mjs", + "upstream-inject-feature-defaults": "bin/upstream-inject-feature-defaults.mjs", "upstream-warmup": "bin/upstream-warmup.mjs" } }, diff --git a/package.json b/package.json index e65e97611..4c5495b58 100644 --- a/package.json +++ b/package.json @@ -26,17 +26,17 @@ }, "devDependencies": { "@arethetypeswrong/cli": "^0.13.0", + "@bufbuild/license-header": "^0.0.4", + "@types/node": "^20.8.8", "@typescript-eslint/eslint-plugin": "^6.9.1", "@typescript-eslint/parser": "^6.9.1", "eslint": "^8.50.0", "eslint-import-resolver-typescript": "^3.6.0", "eslint-plugin-import": "^2.28.0", - "eslint-plugin-node": "^11.1.0", + "eslint-plugin-n": "^16.3.1", + "jest": "^29.7.0", "prettier": "^3.0.0", - "typescript": "^5.2.2", - "@types/node": "^20.8.8", - "@bufbuild/license-header": "^0.0.4", - "jest": "^29.7.0" + "typescript": "^5.2.2" }, "//": "avoid hoisting, see packages/protoplugin/src/ecmascript/transpile.ts", "dependencies": { diff --git a/packages/protobuf-test/src/json-names.test.ts b/packages/protobuf-test/src/json-names.test.ts index 3f743b276..b52fb1394 100644 --- a/packages/protobuf-test/src/json-names.test.ts +++ b/packages/protobuf-test/src/json-names.test.ts @@ -13,115 +13,58 @@ // limitations under the License. import { expect, test } from "@jest/globals"; -import { mkdtempSync, readFileSync, writeFileSync } from "fs"; -import { join } from "path"; -import { tmpdir } from "os"; -import { spawnSync } from "child_process"; import { FileDescriptorSet, proto3, ScalarType } from "@bufbuild/protobuf"; +import { UpstreamProtobuf } from "upstream-protobuf"; -test("JSON names equal protoc", () => { - expect(getProtocJsonName("foo_bar")).toBe("fooBar"); - expectRuntimeJsonNameEqualsProtocJsonName("foo_bar"); - expectRuntimeJsonNameEqualsProtocJsonName("__proto__"); - expectRuntimeJsonNameEqualsProtocJsonName("fieldname1"); - expectRuntimeJsonNameEqualsProtocJsonName("field_name2"); - expectRuntimeJsonNameEqualsProtocJsonName("_field_name3"); - expectRuntimeJsonNameEqualsProtocJsonName("field__name4_"); - expectRuntimeJsonNameEqualsProtocJsonName("field0name5"); - expectRuntimeJsonNameEqualsProtocJsonName("field_0_name6"); - expectRuntimeJsonNameEqualsProtocJsonName("fieldName7"); - expectRuntimeJsonNameEqualsProtocJsonName("FieldName8"); - expectRuntimeJsonNameEqualsProtocJsonName("field_Name9"); - expectRuntimeJsonNameEqualsProtocJsonName("Field_Name10"); - expectRuntimeJsonNameEqualsProtocJsonName("FIELD_NAME11"); - expectRuntimeJsonNameEqualsProtocJsonName("FIELD_name12"); - expectRuntimeJsonNameEqualsProtocJsonName("__field_name13"); - expectRuntimeJsonNameEqualsProtocJsonName("__Field_name14"); - expectRuntimeJsonNameEqualsProtocJsonName("field__name15"); - expectRuntimeJsonNameEqualsProtocJsonName("field__Name16"); - expectRuntimeJsonNameEqualsProtocJsonName("field_name17__"); - expectRuntimeJsonNameEqualsProtocJsonName("Field_name18__"); +test("JSON names equal protoc", async () => { + const names = [ + "foo_bar", + "__proto__", + "fieldname1", + "field_name2", + "_field_name3", + "field__name4_", + "field0name5", + "field_0_name6", + "fieldName7", + "FieldName8", + "field_Name9", + "Field_Name10", + "FIELD_NAME11", + "FIELD_name12", + "__field_name13", + "__Field_name14", + "field__name15", + "field__Name16", + "field_name17__", + "Field_name18__", + ]; + const protocNames = await getProtocJsonNames(names); + const runtimeNames = getRuntimeJsonNames(names); + expect(runtimeNames).toStrictEqual(protocNames); }); -// expectRuntimeJsonNameEqualsProtocJsonName takes the given proto field name -// and runs it through protoc to get the JSON name. -// The result is compared to our own implementation of the algorithm. -// It is important that the implementations in JS and Go match the protoc -// implementation. -export function expectRuntimeJsonNameEqualsProtocJsonName(protoName: string) { - const want = getProtocJsonName(protoName); - const got = getRuntimeJsonName(protoName); - if (want === false) { - return; - } - expect(want).toBe(got); -} - -function getRuntimeJsonName(name: string): string { - const mt = proto3.makeMessageType("Fuzz", [ - { no: 1, kind: "scalar", T: ScalarType.BOOL, name: name }, - ]); - const fi = mt.fields.find(1); - if (!fi) { - throw new Error(); - } - return fi.jsonName; +function getRuntimeJsonNames(protoFieldNames: string[]) { + const mt = proto3.makeMessageType( + "M", + protoFieldNames.map( + (n, i) => + ({ no: i + 1, kind: "scalar", T: ScalarType.INT32, name: n }) as const, + ), + ); + return mt.fields.list().map((f) => f.jsonName); } -// getProtocJsonName runs protoc to get the "json name" for a field -function getProtocJsonName(protoName: string): string | false { - if (protoName.trim() !== protoName) { - return false; - } - const tempDir = mkdtempSync(join(tmpdir(), "node-protoc-workdir-")); - const inFilename = join(tempDir, "i.proto"); - const outFilename = join(tempDir, "o"); - const inData = [ - `syntax = "proto3";`, - `message I {`, - ` int32 ${protoName} = 1;`, - `}`, - ].join("\n"); - writeFileSync(inFilename, inData, { encoding: "utf-8" }); - const result = spawnSync( - "protoc", - ["-I", tempDir, inFilename, "--descriptor_set_out", outFilename], - { - encoding: "utf-8", - }, - ); - if (result.stderr.length > 0) { - if (result.stderr.indexOf("Expected field name.") >= 0) { - return false; - } - if (result.stderr.indexOf("Expected field number.") >= 0) { - return false; - } - if (result.stderr.indexOf('Expected ";".') >= 0) { - return false; - } - if (result.stderr.indexOf("Missing field number.") >= 0) { - return false; - } - if ( - result.stderr.indexOf( - "Invalid control characters encountered in text.", - ) >= 0 - ) { - return false; - } - throw new Error(result.stderr); - } - if (result.error) { - throw result.error; - } - if (result.status !== 0) { - throw new Error("exit code " + String(result.status)); - } - const fds = FileDescriptorSet.fromBinary(readFileSync(outFilename)); - const jsonName = fds.file[0].messageType[0].field[0].jsonName; - if (jsonName === undefined) { - throw new Error("missing json name"); - } - return jsonName; +async function getProtocJsonNames(protoFieldNames: string[]) { + const upstream = new UpstreamProtobuf(); + const bytes = await upstream.compileToDescriptorSet({ + "i.proto": [ + `syntax="proto3";`, + `message M {`, + ...protoFieldNames.map((n, i) => `int32 ${n} = ${i + 1};`), + `}`, + ].join("\n"), + }); + const fds = FileDescriptorSet.fromBinary(bytes); + return fds.file[0].messageType[0].field.map((f) => f.jsonName); } diff --git a/packages/upstream-protobuf/README.md b/packages/upstream-protobuf/README.md index 4282fb408..cf5f98e5d 100644 --- a/packages/upstream-protobuf/README.md +++ b/packages/upstream-protobuf/README.md @@ -1,7 +1,8 @@ # Upstream protobuf -This package provides `protoc`, `conformance_test_runner`, and related proto -files via npm "binaries". +This package provides `protoc`, `conformance_test_runner`, related proto files, +and feature-set defaults for editions via npm "binaries", and via an exported +class. To update this project to use a new version, update the version number in version.txt and run `make`. diff --git a/packages/upstream-protobuf/bin/conformance_test_runner.mjs b/packages/upstream-protobuf/bin/conformance_test_runner.mjs index 5ce68df48..bf62e9c9e 100755 --- a/packages/upstream-protobuf/bin/conformance_test_runner.mjs +++ b/packages/upstream-protobuf/bin/conformance_test_runner.mjs @@ -16,7 +16,7 @@ import { execFileSync } from "node:child_process"; import { argv, exit, stderr } from "node:process"; -import { UpstreamProtobuf } from "../lib.mjs"; +import { UpstreamProtobuf } from "../index.mjs"; const upstream = new UpstreamProtobuf(); diff --git a/packages/upstream-protobuf/bin/protoc.mjs b/packages/upstream-protobuf/bin/protoc.mjs index 87f5cda4a..7124d6537 100755 --- a/packages/upstream-protobuf/bin/protoc.mjs +++ b/packages/upstream-protobuf/bin/protoc.mjs @@ -16,7 +16,7 @@ import { execFileSync } from "node:child_process"; import { argv, exit, stderr } from "node:process"; -import { UpstreamProtobuf } from "../lib.mjs"; +import { UpstreamProtobuf } from "../index.mjs"; const upstream = new UpstreamProtobuf(); diff --git a/packages/upstream-protobuf/bin/upstream-files.mjs b/packages/upstream-protobuf/bin/upstream-files.mjs index d020c8e60..5e95e9486 100755 --- a/packages/upstream-protobuf/bin/upstream-files.mjs +++ b/packages/upstream-protobuf/bin/upstream-files.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {argv, exit, stderr, stdout} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; void main(argv.slice(2)); diff --git a/packages/upstream-protobuf/bin/upstream-include.mjs b/packages/upstream-protobuf/bin/upstream-include.mjs index 3f4927a55..d6aa938a5 100755 --- a/packages/upstream-protobuf/bin/upstream-include.mjs +++ b/packages/upstream-protobuf/bin/upstream-include.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {argv, exit, stderr, stdout} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; void main(argv.slice(2)); diff --git a/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs b/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs new file mode 100755 index 000000000..d8651057e --- /dev/null +++ b/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +// Copyright 2021-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {argv, exit, stdout, stderr} from "node:process"; +import {parseArgs} from "node:util"; +import {readFileSync, writeFileSync} from "node:fs"; +import {UpstreamProtobuf} from "../index.mjs"; + +void main(argv.slice(2)); + +async function main(args) { + let min, max, positionals; + try { + ({values: {min, max}, positionals} = parseArgs({ + args, + options: { + min: { type: 'string' }, + max: { type: 'string' }, + }, + allowPositionals: true, + })); + } catch { + exitUsage(); + } + const upstream = new UpstreamProtobuf(); + const defaults = await upstream.getFeatureSetDefaults(min, max); + stdout.write(`Injecting google.protobuf.FeatureSetDefaults into ${positionals.length} files...\n`); + for (const path of positionals) { + const content = readFileSync(path, "utf-8"); + const r = inject(content, `"${defaults.toString("base64url")}"`); + if (!r.ok) { + stderr.write(`Error injecting into ${path}: ${r.message}\n`); + exit(1); + } + if (r.newContent === content) { + stdout.write(`- ${path} - no changes\n`); + continue; + } + writeFileSync(path, r.newContent); + stdout.write(`- ${path} - updated\n`); + } +} + +/** + * @typedef {object} InjectSuccess + * @property {true} ok + * @property {string} newContent + */ +/** + * @typedef {object} InjectError + * @property {false} ok + * @property {string} message + */ +/** + * @param {string} content + * @param {string} contentToInject + * @return {InjectSuccess | InjectError} + */ +function inject(content, contentToInject) { + const tokenStart = "/*upstream-inject-feature-defaults-start*/"; + const tokenEnd = "/*upstream-inject-feature-defaults-end*/"; + const indexStart = content.indexOf(tokenStart); + const indexEnd = content.indexOf(tokenEnd); + if (indexStart < 0 || indexEnd < 0) { + return {ok: false, message: "missing comment annotations"}; + } + if (indexEnd < indexStart) { + return {ok: false, message: "invalid comment annotations"}; + } + const head = content.substring(0, indexStart + tokenStart.length); + const foot = content.substring(indexEnd); + const newContent = head + contentToInject + foot; + return {ok: true, newContent}; +} + +/** + * @return never + */ +function exitUsage() { + stderr.write(`USAGE: upstream-inject-feature-defaults [--min ] [--max ] \n`); + exit(1); +} diff --git a/packages/upstream-protobuf/bin/upstream-warmup.mjs b/packages/upstream-protobuf/bin/upstream-warmup.mjs index 86a3b16a9..a52f38a9e 100755 --- a/packages/upstream-protobuf/bin/upstream-warmup.mjs +++ b/packages/upstream-protobuf/bin/upstream-warmup.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {exit, stderr} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; const upstream = new UpstreamProtobuf(); upstream.warmup().then(() => exit(0), reason => { diff --git a/packages/upstream-protobuf/index.d.ts b/packages/upstream-protobuf/index.d.ts new file mode 100644 index 000000000..0d0e71577 --- /dev/null +++ b/packages/upstream-protobuf/index.d.ts @@ -0,0 +1,26 @@ +// Copyright 2021-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export declare class UpstreamProtobuf { + constructor(temp?: string, version?: string); + + getProtocPath(): Promise; + + getFeatureSetDefaults( + minimumEdition?: string, + maximumEdition?: string, + ): Promise; + + compileToDescriptorSet(files: Record): Promise; +} diff --git a/packages/upstream-protobuf/lib.mjs b/packages/upstream-protobuf/index.mjs similarity index 86% rename from packages/upstream-protobuf/lib.mjs rename to packages/upstream-protobuf/index.mjs index 6690d503d..6b21f66e9 100644 --- a/packages/upstream-protobuf/lib.mjs +++ b/packages/upstream-protobuf/index.mjs @@ -13,15 +13,18 @@ // limitations under the License. import { + chmodSync, existsSync, - writeFileSync, - readFileSync, mkdirSync, - chmodSync, + mkdtempSync, readdirSync, + readFileSync, + writeFileSync, + rmSync, } from "node:fs"; +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { join as joinPath, dirname, relative as relativePath } from "node:path"; +import { dirname, join as joinPath, relative as relativePath } from "node:path"; import os from "node:os"; import { unzipSync } from "fflate"; import micromatch from "micromatch"; @@ -117,6 +120,71 @@ export class UpstreamProtobuf { ]); } + /** + * @param {Record} files + * @return {Promise} + */ + async compileToDescriptorSet(files) { + const protocPath = await this.getProtocPath(); + const tempDir = mkdtempSync( + joinPath(this.#temp, "compile-descriptor-set-"), + ); + try { + writeTree(Object.entries(files), tempDir); + const outPath = joinPath(tempDir, "desc.bin"); + execFileSync( + protocPath, + [ + "--descriptor_set_out", + outPath, + "--proto_path", + tempDir, + ...Object.keys(files), + ], + { + shell: false, + stdio: "ignore", + }, + ); + return readFileSync(outPath); + } finally { + rmSync(tempDir, { recursive: true }); + } + } + + /** + * @param {string} [minimumEdition] + * @param {string} [maximumEdition] + * @return Promise + */ + async getFeatureSetDefaults(minimumEdition, maximumEdition) { + const binPath = this.#getTempPath( + "feature-set-defaults", + `min-${minimumEdition ?? "default"}-max-${ + maximumEdition ?? "default" + }.bin`, + ); + if (!existsSync(binPath)) { + const protocPath = await this.getProtocPath(); + const args = [ + "--experimental_edition_defaults_out", + binPath, + "google/protobuf/descriptor.proto", + ]; + if (minimumEdition !== undefined) { + args.push("--experimental_edition_defaults_minimum", minimumEdition); + } + if (maximumEdition !== undefined) { + args.push("--experimental_edition_defaults_maximum", maximumEdition); + } + execFileSync(protocPath, args, { + shell: false, + stdio: "ignore", + }); + } + return readFileSync(binPath); + } + /** * @return {Promise} */ @@ -420,7 +488,7 @@ export class UpstreamProtobuf { } /** - * @param {Array<[string, Uint8Array]>} files + * @param {Array<[string, Uint8Array|string]>} files * @param {string} [dir] */ function writeTree(files, dir = ".") { @@ -439,6 +507,7 @@ function writeTree(files, dir = ".") { */ function lsfiles(dir) { const hits = []; + function ls(dir) { for (const ent of readdirSync(dir, { withFileTypes: true })) { const entPath = joinPath(dir, ent.name); @@ -449,6 +518,7 @@ function lsfiles(dir) { } } } + ls(dir); return hits.map((path) => relativePath(dir, path)); } diff --git a/packages/upstream-protobuf/package.json b/packages/upstream-protobuf/package.json index d4e527a39..66c3fb222 100644 --- a/packages/upstream-protobuf/package.json +++ b/packages/upstream-protobuf/package.json @@ -10,7 +10,13 @@ "conformance_test_runner": "bin/conformance_test_runner.mjs", "upstream-files": "bin/upstream-files.mjs", "upstream-include": "bin/upstream-include.mjs", - "upstream-warmup": "bin/upstream-warmup.mjs" + "upstream-warmup": "bin/upstream-warmup.mjs", + "upstream-inject-feature-defaults": "bin/upstream-inject-feature-defaults.mjs" + }, + "exports": { + ".": { + "import": "./index.mjs" + } }, "dependencies": { "fflate": "^0.8.1", From e16130a36d4e00b8c6e24997ff2085760482ce44 Mon Sep 17 00:00:00 2001 From: Timo Stamm Date: Thu, 16 Nov 2023 18:54:36 +0100 Subject: [PATCH 2/2] Make edition feature defaults from `protoc` available --- package-lock.json | 3 +- packages/protobuf-test/src/json-names.test.ts | 153 ++++++------------ packages/upstream-protobuf/README.md | 5 +- .../bin/conformance_test_runner.mjs | 2 +- packages/upstream-protobuf/bin/protoc.mjs | 2 +- .../upstream-protobuf/bin/upstream-files.mjs | 2 +- .../bin/upstream-include.mjs | 2 +- .../bin/upstream-inject-feature-defaults.mjs | 95 +++++++++++ .../upstream-protobuf/bin/upstream-warmup.mjs | 2 +- packages/upstream-protobuf/index.d.ts | 26 +++ .../upstream-protobuf/{lib.mjs => index.mjs} | 80 ++++++++- packages/upstream-protobuf/package.json | 8 +- 12 files changed, 261 insertions(+), 119 deletions(-) create mode 100755 packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs create mode 100644 packages/upstream-protobuf/index.d.ts rename packages/upstream-protobuf/{lib.mjs => index.mjs} (86%) diff --git a/package-lock.json b/package-lock.json index 935a84f9a..c0adf719a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "protobuf-es-6", + "name": "protobuf-es", "lockfileVersion": 3, "requires": true, "packages": { @@ -6443,6 +6443,7 @@ "protoc": "bin/protoc.mjs", "upstream-files": "bin/upstream-files.mjs", "upstream-include": "bin/upstream-include.mjs", + "upstream-inject-feature-defaults": "bin/upstream-inject-feature-defaults.mjs", "upstream-warmup": "bin/upstream-warmup.mjs" } }, diff --git a/packages/protobuf-test/src/json-names.test.ts b/packages/protobuf-test/src/json-names.test.ts index 3f743b276..b52fb1394 100644 --- a/packages/protobuf-test/src/json-names.test.ts +++ b/packages/protobuf-test/src/json-names.test.ts @@ -13,115 +13,58 @@ // limitations under the License. import { expect, test } from "@jest/globals"; -import { mkdtempSync, readFileSync, writeFileSync } from "fs"; -import { join } from "path"; -import { tmpdir } from "os"; -import { spawnSync } from "child_process"; import { FileDescriptorSet, proto3, ScalarType } from "@bufbuild/protobuf"; +import { UpstreamProtobuf } from "upstream-protobuf"; -test("JSON names equal protoc", () => { - expect(getProtocJsonName("foo_bar")).toBe("fooBar"); - expectRuntimeJsonNameEqualsProtocJsonName("foo_bar"); - expectRuntimeJsonNameEqualsProtocJsonName("__proto__"); - expectRuntimeJsonNameEqualsProtocJsonName("fieldname1"); - expectRuntimeJsonNameEqualsProtocJsonName("field_name2"); - expectRuntimeJsonNameEqualsProtocJsonName("_field_name3"); - expectRuntimeJsonNameEqualsProtocJsonName("field__name4_"); - expectRuntimeJsonNameEqualsProtocJsonName("field0name5"); - expectRuntimeJsonNameEqualsProtocJsonName("field_0_name6"); - expectRuntimeJsonNameEqualsProtocJsonName("fieldName7"); - expectRuntimeJsonNameEqualsProtocJsonName("FieldName8"); - expectRuntimeJsonNameEqualsProtocJsonName("field_Name9"); - expectRuntimeJsonNameEqualsProtocJsonName("Field_Name10"); - expectRuntimeJsonNameEqualsProtocJsonName("FIELD_NAME11"); - expectRuntimeJsonNameEqualsProtocJsonName("FIELD_name12"); - expectRuntimeJsonNameEqualsProtocJsonName("__field_name13"); - expectRuntimeJsonNameEqualsProtocJsonName("__Field_name14"); - expectRuntimeJsonNameEqualsProtocJsonName("field__name15"); - expectRuntimeJsonNameEqualsProtocJsonName("field__Name16"); - expectRuntimeJsonNameEqualsProtocJsonName("field_name17__"); - expectRuntimeJsonNameEqualsProtocJsonName("Field_name18__"); +test("JSON names equal protoc", async () => { + const names = [ + "foo_bar", + "__proto__", + "fieldname1", + "field_name2", + "_field_name3", + "field__name4_", + "field0name5", + "field_0_name6", + "fieldName7", + "FieldName8", + "field_Name9", + "Field_Name10", + "FIELD_NAME11", + "FIELD_name12", + "__field_name13", + "__Field_name14", + "field__name15", + "field__Name16", + "field_name17__", + "Field_name18__", + ]; + const protocNames = await getProtocJsonNames(names); + const runtimeNames = getRuntimeJsonNames(names); + expect(runtimeNames).toStrictEqual(protocNames); }); -// expectRuntimeJsonNameEqualsProtocJsonName takes the given proto field name -// and runs it through protoc to get the JSON name. -// The result is compared to our own implementation of the algorithm. -// It is important that the implementations in JS and Go match the protoc -// implementation. -export function expectRuntimeJsonNameEqualsProtocJsonName(protoName: string) { - const want = getProtocJsonName(protoName); - const got = getRuntimeJsonName(protoName); - if (want === false) { - return; - } - expect(want).toBe(got); -} - -function getRuntimeJsonName(name: string): string { - const mt = proto3.makeMessageType("Fuzz", [ - { no: 1, kind: "scalar", T: ScalarType.BOOL, name: name }, - ]); - const fi = mt.fields.find(1); - if (!fi) { - throw new Error(); - } - return fi.jsonName; +function getRuntimeJsonNames(protoFieldNames: string[]) { + const mt = proto3.makeMessageType( + "M", + protoFieldNames.map( + (n, i) => + ({ no: i + 1, kind: "scalar", T: ScalarType.INT32, name: n }) as const, + ), + ); + return mt.fields.list().map((f) => f.jsonName); } -// getProtocJsonName runs protoc to get the "json name" for a field -function getProtocJsonName(protoName: string): string | false { - if (protoName.trim() !== protoName) { - return false; - } - const tempDir = mkdtempSync(join(tmpdir(), "node-protoc-workdir-")); - const inFilename = join(tempDir, "i.proto"); - const outFilename = join(tempDir, "o"); - const inData = [ - `syntax = "proto3";`, - `message I {`, - ` int32 ${protoName} = 1;`, - `}`, - ].join("\n"); - writeFileSync(inFilename, inData, { encoding: "utf-8" }); - const result = spawnSync( - "protoc", - ["-I", tempDir, inFilename, "--descriptor_set_out", outFilename], - { - encoding: "utf-8", - }, - ); - if (result.stderr.length > 0) { - if (result.stderr.indexOf("Expected field name.") >= 0) { - return false; - } - if (result.stderr.indexOf("Expected field number.") >= 0) { - return false; - } - if (result.stderr.indexOf('Expected ";".') >= 0) { - return false; - } - if (result.stderr.indexOf("Missing field number.") >= 0) { - return false; - } - if ( - result.stderr.indexOf( - "Invalid control characters encountered in text.", - ) >= 0 - ) { - return false; - } - throw new Error(result.stderr); - } - if (result.error) { - throw result.error; - } - if (result.status !== 0) { - throw new Error("exit code " + String(result.status)); - } - const fds = FileDescriptorSet.fromBinary(readFileSync(outFilename)); - const jsonName = fds.file[0].messageType[0].field[0].jsonName; - if (jsonName === undefined) { - throw new Error("missing json name"); - } - return jsonName; +async function getProtocJsonNames(protoFieldNames: string[]) { + const upstream = new UpstreamProtobuf(); + const bytes = await upstream.compileToDescriptorSet({ + "i.proto": [ + `syntax="proto3";`, + `message M {`, + ...protoFieldNames.map((n, i) => `int32 ${n} = ${i + 1};`), + `}`, + ].join("\n"), + }); + const fds = FileDescriptorSet.fromBinary(bytes); + return fds.file[0].messageType[0].field.map((f) => f.jsonName); } diff --git a/packages/upstream-protobuf/README.md b/packages/upstream-protobuf/README.md index 4282fb408..cf5f98e5d 100644 --- a/packages/upstream-protobuf/README.md +++ b/packages/upstream-protobuf/README.md @@ -1,7 +1,8 @@ # Upstream protobuf -This package provides `protoc`, `conformance_test_runner`, and related proto -files via npm "binaries". +This package provides `protoc`, `conformance_test_runner`, related proto files, +and feature-set defaults for editions via npm "binaries", and via an exported +class. To update this project to use a new version, update the version number in version.txt and run `make`. diff --git a/packages/upstream-protobuf/bin/conformance_test_runner.mjs b/packages/upstream-protobuf/bin/conformance_test_runner.mjs index 5ce68df48..bf62e9c9e 100755 --- a/packages/upstream-protobuf/bin/conformance_test_runner.mjs +++ b/packages/upstream-protobuf/bin/conformance_test_runner.mjs @@ -16,7 +16,7 @@ import { execFileSync } from "node:child_process"; import { argv, exit, stderr } from "node:process"; -import { UpstreamProtobuf } from "../lib.mjs"; +import { UpstreamProtobuf } from "../index.mjs"; const upstream = new UpstreamProtobuf(); diff --git a/packages/upstream-protobuf/bin/protoc.mjs b/packages/upstream-protobuf/bin/protoc.mjs index 87f5cda4a..7124d6537 100755 --- a/packages/upstream-protobuf/bin/protoc.mjs +++ b/packages/upstream-protobuf/bin/protoc.mjs @@ -16,7 +16,7 @@ import { execFileSync } from "node:child_process"; import { argv, exit, stderr } from "node:process"; -import { UpstreamProtobuf } from "../lib.mjs"; +import { UpstreamProtobuf } from "../index.mjs"; const upstream = new UpstreamProtobuf(); diff --git a/packages/upstream-protobuf/bin/upstream-files.mjs b/packages/upstream-protobuf/bin/upstream-files.mjs index d020c8e60..5e95e9486 100755 --- a/packages/upstream-protobuf/bin/upstream-files.mjs +++ b/packages/upstream-protobuf/bin/upstream-files.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {argv, exit, stderr, stdout} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; void main(argv.slice(2)); diff --git a/packages/upstream-protobuf/bin/upstream-include.mjs b/packages/upstream-protobuf/bin/upstream-include.mjs index 3f4927a55..d6aa938a5 100755 --- a/packages/upstream-protobuf/bin/upstream-include.mjs +++ b/packages/upstream-protobuf/bin/upstream-include.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {argv, exit, stderr, stdout} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; void main(argv.slice(2)); diff --git a/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs b/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs new file mode 100755 index 000000000..d8651057e --- /dev/null +++ b/packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +// Copyright 2021-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {argv, exit, stdout, stderr} from "node:process"; +import {parseArgs} from "node:util"; +import {readFileSync, writeFileSync} from "node:fs"; +import {UpstreamProtobuf} from "../index.mjs"; + +void main(argv.slice(2)); + +async function main(args) { + let min, max, positionals; + try { + ({values: {min, max}, positionals} = parseArgs({ + args, + options: { + min: { type: 'string' }, + max: { type: 'string' }, + }, + allowPositionals: true, + })); + } catch { + exitUsage(); + } + const upstream = new UpstreamProtobuf(); + const defaults = await upstream.getFeatureSetDefaults(min, max); + stdout.write(`Injecting google.protobuf.FeatureSetDefaults into ${positionals.length} files...\n`); + for (const path of positionals) { + const content = readFileSync(path, "utf-8"); + const r = inject(content, `"${defaults.toString("base64url")}"`); + if (!r.ok) { + stderr.write(`Error injecting into ${path}: ${r.message}\n`); + exit(1); + } + if (r.newContent === content) { + stdout.write(`- ${path} - no changes\n`); + continue; + } + writeFileSync(path, r.newContent); + stdout.write(`- ${path} - updated\n`); + } +} + +/** + * @typedef {object} InjectSuccess + * @property {true} ok + * @property {string} newContent + */ +/** + * @typedef {object} InjectError + * @property {false} ok + * @property {string} message + */ +/** + * @param {string} content + * @param {string} contentToInject + * @return {InjectSuccess | InjectError} + */ +function inject(content, contentToInject) { + const tokenStart = "/*upstream-inject-feature-defaults-start*/"; + const tokenEnd = "/*upstream-inject-feature-defaults-end*/"; + const indexStart = content.indexOf(tokenStart); + const indexEnd = content.indexOf(tokenEnd); + if (indexStart < 0 || indexEnd < 0) { + return {ok: false, message: "missing comment annotations"}; + } + if (indexEnd < indexStart) { + return {ok: false, message: "invalid comment annotations"}; + } + const head = content.substring(0, indexStart + tokenStart.length); + const foot = content.substring(indexEnd); + const newContent = head + contentToInject + foot; + return {ok: true, newContent}; +} + +/** + * @return never + */ +function exitUsage() { + stderr.write(`USAGE: upstream-inject-feature-defaults [--min ] [--max ] \n`); + exit(1); +} diff --git a/packages/upstream-protobuf/bin/upstream-warmup.mjs b/packages/upstream-protobuf/bin/upstream-warmup.mjs index 86a3b16a9..a52f38a9e 100755 --- a/packages/upstream-protobuf/bin/upstream-warmup.mjs +++ b/packages/upstream-protobuf/bin/upstream-warmup.mjs @@ -15,7 +15,7 @@ // limitations under the License. import {exit, stderr} from "node:process"; -import {UpstreamProtobuf} from "../lib.mjs"; +import {UpstreamProtobuf} from "../index.mjs"; const upstream = new UpstreamProtobuf(); upstream.warmup().then(() => exit(0), reason => { diff --git a/packages/upstream-protobuf/index.d.ts b/packages/upstream-protobuf/index.d.ts new file mode 100644 index 000000000..0d0e71577 --- /dev/null +++ b/packages/upstream-protobuf/index.d.ts @@ -0,0 +1,26 @@ +// Copyright 2021-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export declare class UpstreamProtobuf { + constructor(temp?: string, version?: string); + + getProtocPath(): Promise; + + getFeatureSetDefaults( + minimumEdition?: string, + maximumEdition?: string, + ): Promise; + + compileToDescriptorSet(files: Record): Promise; +} diff --git a/packages/upstream-protobuf/lib.mjs b/packages/upstream-protobuf/index.mjs similarity index 86% rename from packages/upstream-protobuf/lib.mjs rename to packages/upstream-protobuf/index.mjs index 6690d503d..6b21f66e9 100644 --- a/packages/upstream-protobuf/lib.mjs +++ b/packages/upstream-protobuf/index.mjs @@ -13,15 +13,18 @@ // limitations under the License. import { + chmodSync, existsSync, - writeFileSync, - readFileSync, mkdirSync, - chmodSync, + mkdtempSync, readdirSync, + readFileSync, + writeFileSync, + rmSync, } from "node:fs"; +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { join as joinPath, dirname, relative as relativePath } from "node:path"; +import { dirname, join as joinPath, relative as relativePath } from "node:path"; import os from "node:os"; import { unzipSync } from "fflate"; import micromatch from "micromatch"; @@ -117,6 +120,71 @@ export class UpstreamProtobuf { ]); } + /** + * @param {Record} files + * @return {Promise} + */ + async compileToDescriptorSet(files) { + const protocPath = await this.getProtocPath(); + const tempDir = mkdtempSync( + joinPath(this.#temp, "compile-descriptor-set-"), + ); + try { + writeTree(Object.entries(files), tempDir); + const outPath = joinPath(tempDir, "desc.bin"); + execFileSync( + protocPath, + [ + "--descriptor_set_out", + outPath, + "--proto_path", + tempDir, + ...Object.keys(files), + ], + { + shell: false, + stdio: "ignore", + }, + ); + return readFileSync(outPath); + } finally { + rmSync(tempDir, { recursive: true }); + } + } + + /** + * @param {string} [minimumEdition] + * @param {string} [maximumEdition] + * @return Promise + */ + async getFeatureSetDefaults(minimumEdition, maximumEdition) { + const binPath = this.#getTempPath( + "feature-set-defaults", + `min-${minimumEdition ?? "default"}-max-${ + maximumEdition ?? "default" + }.bin`, + ); + if (!existsSync(binPath)) { + const protocPath = await this.getProtocPath(); + const args = [ + "--experimental_edition_defaults_out", + binPath, + "google/protobuf/descriptor.proto", + ]; + if (minimumEdition !== undefined) { + args.push("--experimental_edition_defaults_minimum", minimumEdition); + } + if (maximumEdition !== undefined) { + args.push("--experimental_edition_defaults_maximum", maximumEdition); + } + execFileSync(protocPath, args, { + shell: false, + stdio: "ignore", + }); + } + return readFileSync(binPath); + } + /** * @return {Promise} */ @@ -420,7 +488,7 @@ export class UpstreamProtobuf { } /** - * @param {Array<[string, Uint8Array]>} files + * @param {Array<[string, Uint8Array|string]>} files * @param {string} [dir] */ function writeTree(files, dir = ".") { @@ -439,6 +507,7 @@ function writeTree(files, dir = ".") { */ function lsfiles(dir) { const hits = []; + function ls(dir) { for (const ent of readdirSync(dir, { withFileTypes: true })) { const entPath = joinPath(dir, ent.name); @@ -449,6 +518,7 @@ function lsfiles(dir) { } } } + ls(dir); return hits.map((path) => relativePath(dir, path)); } diff --git a/packages/upstream-protobuf/package.json b/packages/upstream-protobuf/package.json index d4e527a39..66c3fb222 100644 --- a/packages/upstream-protobuf/package.json +++ b/packages/upstream-protobuf/package.json @@ -10,7 +10,13 @@ "conformance_test_runner": "bin/conformance_test_runner.mjs", "upstream-files": "bin/upstream-files.mjs", "upstream-include": "bin/upstream-include.mjs", - "upstream-warmup": "bin/upstream-warmup.mjs" + "upstream-warmup": "bin/upstream-warmup.mjs", + "upstream-inject-feature-defaults": "bin/upstream-inject-feature-defaults.mjs" + }, + "exports": { + ".": { + "import": "./index.mjs" + } }, "dependencies": { "fflate": "^0.8.1",