Make edition feature defaults from protoc available

packages/protobuf-test/src/json-names.test.ts

@@ -13,115 +13,58 @@
 // limitations under the License.
 
 import { expect, test } from "@jest/globals";
-import { mkdtempSync, readFileSync, writeFileSync } from "fs";
-import { join } from "path";
-import { tmpdir } from "os";
-import { spawnSync } from "child_process";
 import { FileDescriptorSet, proto3, ScalarType } from "@bufbuild/protobuf";
+import { UpstreamProtobuf } from "upstream-protobuf";
 
-test("JSON names equal protoc", () => {
-  expect(getProtocJsonName("foo_bar")).toBe("fooBar");
-  expectRuntimeJsonNameEqualsProtocJsonName("foo_bar");
-  expectRuntimeJsonNameEqualsProtocJsonName("__proto__");
-  expectRuntimeJsonNameEqualsProtocJsonName("fieldname1");
-  expectRuntimeJsonNameEqualsProtocJsonName("field_name2");
-  expectRuntimeJsonNameEqualsProtocJsonName("_field_name3");
-  expectRuntimeJsonNameEqualsProtocJsonName("field__name4_");
-  expectRuntimeJsonNameEqualsProtocJsonName("field0name5");
-  expectRuntimeJsonNameEqualsProtocJsonName("field_0_name6");
-  expectRuntimeJsonNameEqualsProtocJsonName("fieldName7");
-  expectRuntimeJsonNameEqualsProtocJsonName("FieldName8");
-  expectRuntimeJsonNameEqualsProtocJsonName("field_Name9");
-  expectRuntimeJsonNameEqualsProtocJsonName("Field_Name10");
-  expectRuntimeJsonNameEqualsProtocJsonName("FIELD_NAME11");
-  expectRuntimeJsonNameEqualsProtocJsonName("FIELD_name12");
-  expectRuntimeJsonNameEqualsProtocJsonName("__field_name13");
-  expectRuntimeJsonNameEqualsProtocJsonName("__Field_name14");
-  expectRuntimeJsonNameEqualsProtocJsonName("field__name15");
-  expectRuntimeJsonNameEqualsProtocJsonName("field__Name16");
-  expectRuntimeJsonNameEqualsProtocJsonName("field_name17__");
-  expectRuntimeJsonNameEqualsProtocJsonName("Field_name18__");
+test("JSON names equal protoc", async () => {
+  const names = [
+    "foo_bar",
+    "__proto__",
+    "fieldname1",
+    "field_name2",
+    "_field_name3",
+    "field__name4_",
+    "field0name5",
+    "field_0_name6",
+    "fieldName7",
+    "FieldName8",
+    "field_Name9",
+    "Field_Name10",
+    "FIELD_NAME11",
+    "FIELD_name12",
+    "__field_name13",
+    "__Field_name14",
+    "field__name15",
+    "field__Name16",
+    "field_name17__",
+    "Field_name18__",
+  ];
+  const protocNames = await getProtocJsonNames(names);
+  const runtimeNames = getRuntimeJsonNames(names);
+  expect(runtimeNames).toStrictEqual(protocNames);
 });
 
-// expectRuntimeJsonNameEqualsProtocJsonName takes the given proto field name
-// and runs it through protoc to get the JSON name.
-// The result is compared to our own implementation of the algorithm.
-// It is important that the implementations in JS and Go match the protoc
-// implementation.
-export function expectRuntimeJsonNameEqualsProtocJsonName(protoName: string) {
-  const want = getProtocJsonName(protoName);
-  const got = getRuntimeJsonName(protoName);
-  if (want === false) {
-    return;
-  }
-  expect(want).toBe(got);
-}
-
-function getRuntimeJsonName(name: string): string {
-  const mt = proto3.makeMessageType("Fuzz", [
-    { no: 1, kind: "scalar", T: ScalarType.BOOL, name: name },
-  ]);
-  const fi = mt.fields.find(1);
-  if (!fi) {
-    throw new Error();
-  }
-  return fi.jsonName;
+function getRuntimeJsonNames(protoFieldNames: string[]) {
+  const mt = proto3.makeMessageType(
+    "M",
+    protoFieldNames.map(
+      (n, i) =>
+        ({ no: i + 1, kind: "scalar", T: ScalarType.INT32, name: n }) as const,
+    ),
+  );
+  return mt.fields.list().map((f) => f.jsonName);
 }
 
-// getProtocJsonName runs protoc to get the "json name" for a field
-function getProtocJsonName(protoName: string): string | false {
-  if (protoName.trim() !== protoName) {
-    return false;
-  }
-  const tempDir = mkdtempSync(join(tmpdir(), "node-protoc-workdir-"));
-  const inFilename = join(tempDir, "i.proto");
-  const outFilename = join(tempDir, "o");
-  const inData = [
-    `syntax = "proto3";`,
-    `message I {`,
-    `    int32 ${protoName} = 1;`,
-    `}`,
-  ].join("\n");
-  writeFileSync(inFilename, inData, { encoding: "utf-8" });
-  const result = spawnSync(
-    "protoc",
-    ["-I", tempDir, inFilename, "--descriptor_set_out", outFilename],
-    {
-      encoding: "utf-8",
-    },
-  );
-  if (result.stderr.length > 0) {
-    if (result.stderr.indexOf("Expected field name.") >= 0) {
-      return false;
-    }
-    if (result.stderr.indexOf("Expected field number.") >= 0) {
-      return false;
-    }
-    if (result.stderr.indexOf('Expected ";".') >= 0) {
-      return false;
-    }
-    if (result.stderr.indexOf("Missing field number.") >= 0) {
-      return false;
-    }
-    if (
-      result.stderr.indexOf(
-        "Invalid control characters encountered in text.",
-      ) >= 0
-    ) {
-      return false;
-    }
-    throw new Error(result.stderr);
-  }
-  if (result.error) {
-    throw result.error;
-  }
-  if (result.status !== 0) {
-    throw new Error("exit code " + String(result.status));
-  }
-  const fds = FileDescriptorSet.fromBinary(readFileSync(outFilename));
-  const jsonName = fds.file[0].messageType[0].field[0].jsonName;
-  if (jsonName === undefined) {
-    throw new Error("missing json name");
-  }
-  return jsonName;
+async function getProtocJsonNames(protoFieldNames: string[]) {
+  const upstream = new UpstreamProtobuf();
+  const bytes = await upstream.compileToDescriptorSet({
+    "i.proto": [
+      `syntax="proto3";`,
+      `message M {`,
+      ...protoFieldNames.map((n, i) => `int32 ${n} = ${i + 1};`),
+      `}`,
+    ].join("\n"),
+  });
+  const fds = FileDescriptorSet.fromBinary(bytes);
+  return fds.file[0].messageType[0].field.map((f) => f.jsonName);
 }

packages/upstream-protobuf/README.md

@@ -1,7 +1,8 @@
 # Upstream protobuf
 
-This package provides `protoc`, `conformance_test_runner`, and related proto
-files via npm "binaries".
+This package provides `protoc`, `conformance_test_runner`, related proto files, 
+and feature-set defaults for editions via npm "binaries", and via an exported 
+class.
 
 To update this project to use a new version, update the version number in 
 version.txt and run `make`.

packages/upstream-protobuf/bin/conformance_test_runner.mjs

@@ -16,7 +16,7 @@
 
 import { execFileSync } from "node:child_process";
 import { argv, exit, stderr } from "node:process";
-import { UpstreamProtobuf } from "../lib.mjs";
+import { UpstreamProtobuf } from "../index.mjs";
 
 const upstream = new UpstreamProtobuf();
 

packages/upstream-protobuf/bin/protoc.mjs

@@ -16,7 +16,7 @@
 
 import { execFileSync } from "node:child_process";
 import { argv, exit, stderr } from "node:process";
-import { UpstreamProtobuf } from "../lib.mjs";
+import { UpstreamProtobuf } from "../index.mjs";
 
 const upstream = new UpstreamProtobuf();
 

packages/upstream-protobuf/bin/upstream-files.mjs

@@ -15,7 +15,7 @@
 // limitations under the License.
 
 import {argv, exit, stderr, stdout} from "node:process";
-import {UpstreamProtobuf} from "../lib.mjs";
+import {UpstreamProtobuf} from "../index.mjs";
 
 void main(argv.slice(2));
 

packages/upstream-protobuf/bin/upstream-include.mjs

@@ -15,7 +15,7 @@
 // limitations under the License.
 
 import {argv, exit, stderr, stdout} from "node:process";
-import {UpstreamProtobuf} from "../lib.mjs";
+import {UpstreamProtobuf} from "../index.mjs";
 
 void main(argv.slice(2));
 

packages/upstream-protobuf/bin/upstream-inject-feature-defaults.mjs

@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+
+// Copyright 2021-2023 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {argv, exit, stdout, stderr} from "node:process";
+import {parseArgs} from "node:util";
+import {readFileSync, writeFileSync} from "node:fs";
+import {UpstreamProtobuf} from "../index.mjs";
+
+void main(argv.slice(2));
+
+async function main(args) {
+    let min, max, positionals;
+    try {
+        ({values: {min, max}, positionals} = parseArgs({
+            args,
+            options: {
+                min: { type: 'string' },
+                max: { type: 'string' },
+            },
+            allowPositionals: true,
+        }));
+    } catch {
+        exitUsage();
+    }
+    const upstream = new UpstreamProtobuf();
+    const defaults = await upstream.getFeatureSetDefaults(min, max);
+    stdout.write(`Injecting google.protobuf.FeatureSetDefaults into ${positionals.length} files...\n`);
+    for (const path of positionals) {
+        const content = readFileSync(path, "utf-8");
+        const r = inject(content, `"${defaults.toString("base64url")}"`);
+        if (!r.ok) {
+            stderr.write(`Error injecting into ${path}: ${r.message}\n`);
+            exit(1);
+        }
+        if (r.newContent === content) {
+            stdout.write(`- ${path} - no changes\n`);
+            continue;
+        }
+        writeFileSync(path, r.newContent);
+        stdout.write(`- ${path} - updated\n`);
+    }
+}
+
+/**
+ * @typedef {object} InjectSuccess
+ * @property {true} ok
+ * @property {string} newContent
+ */
+/**
+ * @typedef {object} InjectError
+ * @property {false} ok
+ * @property {string} message
+ */
+/**
+ * @param {string} content
+ * @param {string} contentToInject
+ * @return {InjectSuccess | InjectError}
+ */
+function inject(content, contentToInject) {
+    const tokenStart = "/*upstream-inject-feature-defaults-start*/";
+    const tokenEnd = "/*upstream-inject-feature-defaults-end*/";
+    const indexStart = content.indexOf(tokenStart);
+    const indexEnd = content.indexOf(tokenEnd);
+    if (indexStart < 0 || indexEnd < 0) {
+        return {ok: false, message: "missing comment annotations"};
+    }
+    if (indexEnd < indexStart) {
+        return {ok: false, message: "invalid comment annotations"};
+    }
+    const head = content.substring(0, indexStart + tokenStart.length);
+    const foot = content.substring(indexEnd);
+    const newContent = head + contentToInject + foot;
+    return {ok: true, newContent};
+}
+
+/**
+ * @return never
+ */
+function exitUsage() {
+    stderr.write(`USAGE: upstream-inject-feature-defaults [--min <mininum supported edition>] [--max <maximum supported edition>] <file-to-inject-into>\n`);
+    exit(1);
+}

packages/upstream-protobuf/bin/upstream-warmup.mjs

@@ -15,7 +15,7 @@
 // limitations under the License.
 
 import {exit, stderr} from "node:process";
-import {UpstreamProtobuf} from "../lib.mjs";
+import {UpstreamProtobuf} from "../index.mjs";
 
 const upstream = new UpstreamProtobuf();
 upstream.warmup().then(() => exit(0), reason => {

packages/upstream-protobuf/index.d.ts

@@ -0,0 +1,26 @@
+// Copyright 2021-2023 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export declare class UpstreamProtobuf {
+  constructor(temp?: string, version?: string);
+
+  getProtocPath(): Promise<string>;
+
+  getFeatureSetDefaults(
+    minimumEdition?: string,
+    maximumEdition?: string,
+  ): Promise<Uint8Array>;
+
+  compileToDescriptorSet(files: Record<string, string>): Promise<Uint8Array>;
+}