diff --git a/.bazelignore b/.bazelignore index 4627aa799..729d05bdd 100644 --- a/.bazelignore +++ b/.bazelignore @@ -23,3 +23,4 @@ node_modules/ npm/private/test/node_modules/ npm/private/test/npm_package/node_modules/ npm/private/test/npm_package_publish/node_modules +.git diff --git a/npm/private/npm_translate_lock_generate.bzl b/npm/private/npm_translate_lock_generate.bzl index 977c566a6..b17e046b4 100644 --- a/npm/private/npm_translate_lock_generate.bzl +++ b/npm/private/npm_translate_lock_generate.bzl @@ -166,7 +166,7 @@ sh_binary( dep_path = helpers.link_package(root_package, dep_version[len("file:"):]) dep_key = "{}+{}".format(dep_package, dep_version) if not dep_key in fp_links.keys(): - msg = "Expected to file: referenced package {} in first-party links".format(dep_key) + msg = "Expected to find: referenced package {} in first-party links".format(dep_key) fail(msg) fp_links[dep_key]["link_packages"][link_package] = True elif dep_version.startswith("link:"): diff --git a/npm/private/pnpm.bzl b/npm/private/pnpm.bzl index 86f352619..8999349ae 100644 --- a/npm/private/pnpm.bzl +++ b/npm/private/pnpm.bzl @@ -28,11 +28,33 @@ def _new_import_info(dependencies, dev_dependencies, optional_dependencies): "optional_dependencies": optional_dependencies, } -# Metadata about a package. -# -# Metadata may come from different locations depending on the lockfile, this struct should -# have data normalized across lockfiles. def _new_package_info(id, name, dependencies, optional_dependencies, dev, has_bin, optional, requires_build, version, friendly_version, resolution): + """ + Metadata about a package. + + Metadata may come from different locations depending on the lockfile, this struct should + have data normalized across lockfiles. + + Args: + id: The package id, if present. + TODO Remove. Used for to resolve path of local packages, however `resolution` is a better source of truth. + name: The package name. + dependencies: A map of package dependencies. + optional_dependencies: A map of optional package dependencies. + dev: True if the package is a dev dependency, None otherwise. + has_bin: True if the package has a bin field. + optional: True if the package is an optional dependency. + Determines if package should be omitted `no_optional = True` specified. + requires_build: True if the package requires a build. + NOTE: With pnpm v9, this cannot be known ahead of time. + version: The resolved package version. + e.g. `file:packages/a`, `1.2.3`, `1.2.3_at_scope_peer_2.0.2`. + friendly_version: The package version, normalized for users. Used to target patches, etc. + e.g. `file:packages/a`, `1.2.3`. + resolution: The package resolution. + e.g. { integrity: "..." } + e.g. { type: "directory", directory: "packages/a" } + """ return { "id": id, "name": name, @@ -218,20 +240,20 @@ def _convert_pnpm_v6_v9_version_peer_dep(version): # with rules_js. # # Examples: - # 1.2.3 - # 1.2.3(@scope/peer@2.0.2)(@scope/peer@4.5.6) - # 4.5.6(patch_hash=o3deharooos255qt5xdujc3cuq) + # 1.2.3 -> 1.2.3 + # 1.2.3(@scope/peer@2.0.2)(@scope/peer@4.5.6) -> 1.2.3_2001974805 + # 4.5.6(patch_hash=o3deharooos255qt5xdujc3cuq) -> 4.5.6_o3deharooos255qt5xdujc3cuq if version[-1] == ")": # Drop the patch_hash= not present in v5 so (patch_hash=123) -> (123) like v5 version = version.replace("(patch_hash=", "(") - # There is a peer dep if the string ends with ")" + # There is a peer dep (or patch) if the string ends with ")" peer_dep_index = version.find("(") peer_dep = version[peer_dep_index:] if len(peer_dep) > 32: # Prevent long paths. The pnpm lockfile v6 no longer hashes long sequences of # peer deps so we must hash here to prevent extremely long file paths that lead to - # "File name too long) build failures. + # "File name too long" build failures. peer_dep = utils.hash(peer_dep) else: peer_dep = peer_dep.replace("(@", "(_at_").replace(")(", "_").replace("@", "_").replace("/", "_") @@ -603,4 +625,5 @@ pnpm = struct( # Exported only to be tested pnpm_test = struct( strip_v5_peer_dep_or_patched_version = _strip_v5_peer_dep_or_patched_version, + convert_pnpm_v6_v9_version_peer_dep = _convert_pnpm_v6_v9_version_peer_dep, ) diff --git a/npm/private/test/parse_pnpm_lock_tests.bzl b/npm/private/test/parse_pnpm_lock_tests.bzl index a24bab5b7..28ee76803 100644 --- a/npm/private/test/parse_pnpm_lock_tests.bzl +++ b/npm/private/test/parse_pnpm_lock_tests.bzl @@ -46,34 +46,92 @@ expected_packages = { }, } +expected_imports_injected = { + ".": { + "dependencies": {}, + "dev_dependencies": {}, + "optional_dependencies": {}, + }, + "packages/a": { + "dependencies": { + "b": "file:packages/b_typescript_5.6.2", + }, + "dev_dependencies": {}, + "optional_dependencies": {}, + }, + "packages/b": { + "dependencies": { + "typescript": "5.6.2", + }, + "dev_dependencies": {}, + "optional_dependencies": {}, + }, +} +expected_packages_injected = { + "file:packages/b_typescript_5.6.2": { + "id": "file:packages/b", + "name": "b", + "dependencies": { + "typescript": "5.6.2", + }, + "optional_dependencies": {}, + "dev": False, + "has_bin": False, + "optional": False, + "requires_build": False, + "version": "file:packages/b_typescript_5.6.2", + "friendly_version": "file:packages/b_typescript_5.6.2", + "resolution": { + "directory": "packages/b", + "type": "directory", + }, + }, + "typescript@5.6.2": { + "id": None, + "name": "typescript", + "dependencies": {}, + "optional_dependencies": {}, + "dev": False, + "has_bin": True, + "optional": False, + "requires_build": False, + "version": "5.6.2", + "friendly_version": "5.6.2", + "resolution": { + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + }, + }, +} + +# Example: https://github.com/pnpm/pnpm/blob/0672517f694da62dff7c33b9e723fbfb036eaefa/pnpm-lock.yaml def _parse_lockfile_v5_test_impl(ctx): env = unittest.begin(ctx) - parsed_json = pnpm.parse_pnpm_lock_json("""\ -{ - "lockfileVersion": 5.4, - "specifiers": { - "@aspect-test/a": "5.0.0" - }, - "dependencies": { - "@aspect-test/a": "5.0.0" - }, - "packages": { - "/@aspect-test/a/5.0.0": { - "resolution": { - "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==" - }, - "hasBin": true, - "dependencies": { - "@aspect-test/b": "5.0.0", - "@aspect-test/c": "1.0.0", - "@aspect-test/d": "2.0.0_@aspect-test+c@1.0.0" - }, - "dev": false - } - } -} -""") + parsed_json = pnpm.parse_pnpm_lock_json(json.encode({ + "lockfileVersion": 5.4, + "specifiers": { + "@aspect-test/a": "5.0.0", + }, + "dependencies": { + "@aspect-test/a": "5.0.0", + }, + "packages": { + "/@aspect-test/a/5.0.0": { + "resolution": { + "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", + }, + "hasBin": True, + "dependencies": { + # TODO Test data defect, all listed dependencies must have a definition + "@aspect-test/b": "5.0.0", + "@aspect-test/c": "1.0.0", + # Package has 1 peer dependency (`@aspect-test/c`), in v5 packages with more than 1 use a hash instead + "@aspect-test/d": "2.0.0_@aspect-test+c@1.0.0", + }, + "dev": False, + }, + }, + })) expected = ( expected_importers, @@ -90,31 +148,32 @@ def _parse_lockfile_v5_test_impl(ctx): def _parse_lockfile_v6_test_impl(ctx): env = unittest.begin(ctx) - parsed_json = pnpm.parse_pnpm_lock_json("""\ -{ - "lockfileVersion": "6.0", - "dependencies": { - "@aspect-test/a": { - "specifier": "5.0.0", - "version": "5.0.0" - } - }, - "packages": { - "/@aspect-test/a@5.0.0": { - "resolution": { - "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==" - }, - "hasBin": true, - "dependencies": { - "@aspect-test/b": "5.0.0", - "@aspect-test/c": "1.0.0", - "@aspect-test/d": "2.0.0(@aspect-test/c@1.0.0)" - }, - "dev": false - } - } -} -""") + parsed_json = pnpm.parse_pnpm_lock_json(json.encode({ + "lockfileVersion": "6.0", + "dependencies": { + "@aspect-test/a": { + "specifier": "5.0.0", + "version": "5.0.0", + }, + }, + "packages": { + "/@aspect-test/a@5.0.0": { + "resolution": { + "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", + }, + "hasBin": True, + "dependencies": { + # TODO Test data defect, all listed dependencies must have a definition + "@aspect-test/b": "5.0.0", + "@aspect-test/c": "1.0.0", + # Package has 1 peer dependency (`@aspect-test/c`), packages with several peer dependencies may be given a hash (to satisfy Windows path length limits) + # `npm_translate_lock` will likewise replace the peer dependency component with a hash if too long. + "@aspect-test/d": "2.0.0(@aspect-test/c@1.0.0)", + }, + "dev": False, + }, + }, + })) expected = ( expected_importers, @@ -128,51 +187,127 @@ def _parse_lockfile_v6_test_impl(ctx): return unittest.end(env) +def _parse_lockfile_v6_local_injected_test_impl(ctx): + env = unittest.begin(ctx) + + parsed_json = pnpm.parse_pnpm_lock_json(json.encode({ + "lockfileVersion": "6.0", + "settings": { + "autoInstallPeers": True, + "excludeLinksFromLockfile": False, + }, + "importers": { + ".": {}, + "packages/a": { + "dependencies": { + "b": { + "specifier": "workspace:*", + "version": "file:packages/b(typescript@5.6.2)", + }, + }, + "dependenciesMeta": { + "b": { + "injected": True, + }, + }, + }, + "packages/b": { + "dependencies": { + "typescript": { + "specifier": "^5.6.2", + "version": "5.6.2", + }, + }, + }, + }, + "packages": { + "/typescript@5.6.2": { + "resolution": { + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + }, + "engines": { + "node": ">=14.17", + }, + "hasBin": True, + "dev": False, + }, + "file:packages/b(typescript@5.6.2)": { + "resolution": { + "directory": "packages/b", + "type": "directory", + }, + "id": "file:packages/b", + "name": "b", + "peerDependencies": { + "typescript": "^5.6.2", + }, + "dependencies": { + "typescript": "5.6.2", + }, + "dev": False, + }, + }, + })) + + expected = ( + expected_imports_injected, + expected_packages_injected, + {}, + 6.0, + None, + ) + + asserts.equals(env, expected, parsed_json) + + return unittest.end(env) + def _parse_lockfile_v9_test_impl(ctx): env = unittest.begin(ctx) - parsed_json = pnpm.parse_pnpm_lock_json("""\ -{ - "lockfileVersion": "9.0", - "settings": { - "autoInstallPeers": true, - "excludeLinksFromLockfile": false - }, - "importers": { - ".": { - "dependencies": { - "@aspect-test/a": { - "specifier": "5.0.0", - "version": "5.0.0" - } - } - } - }, - "packages": { - "@aspect-test/a@5.0.0": { - "resolution": { - "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==" - }, - "hasBin": true - } - }, - "snapshots": { - "@aspect-test/a@5.0.0": { - "dependencies": { - "@aspect-test/b": "5.0.0", - "@aspect-test/c": "1.0.0", - "@aspect-test/d": "2.0.0(@aspect-test/c@1.0.0)" - } - } - } -} -""") + parsed_json = pnpm.parse_pnpm_lock_json(json.encode({ + "lockfileVersion": "9.0", + "settings": { + "autoInstallPeers": True, + "excludeLinksFromLockfile": False, + }, + "importers": { + ".": { + "dependencies": { + "@aspect-test/a": { + "specifier": "5.0.0", + "version": "5.0.0", + }, + }, + }, + }, + "packages": { + "@aspect-test/a@5.0.0": { + "resolution": { + "integrity": "sha512-t/lwpVXG/jmxTotGEsmjwuihC2Lvz/Iqt63o78SI3O5XallxtFp5j2WM2M6HwkFiii9I42KdlAF8B3plZMz0Fw==", + }, + "hasBin": True, + }, + }, + "snapshots": { + "@aspect-test/a@5.0.0": { + "dependencies": { + # TODO Test data defect, all listed dependencies must have a definition + "@aspect-test/b": "5.0.0", + "@aspect-test/c": "1.0.0", + # Package has 1 peer dependency (`@aspect-test/c`), packages with several peer dependencies may be given a hash (to satisfy Windows path length limits) + # `npm_translate_lock` will likewise replace the peer dependency component with a hash if too long. + "@aspect-test/d": "2.0.0(@aspect-test/c@1.0.0)", + }, + }, + }, + })) # NOTE: unknown properties in >=v9 v9_expected_packages = dict(expected_packages) - v9_expected_packages["@aspect-test/a@5.0.0"] = dict(v9_expected_packages["@aspect-test/a@5.0.0"]) - v9_expected_packages["@aspect-test/a@5.0.0"]["dev"] = None - v9_expected_packages["@aspect-test/a@5.0.0"]["requires_build"] = None + for pkg_name in v9_expected_packages.keys(): + v9_expected_packages[pkg_name] = dict(v9_expected_packages[pkg_name]) + v9_expected_packages[pkg_name]["dev"] = None + v9_expected_packages[pkg_name]["requires_build"] = None expected = ( expected_importers, @@ -186,6 +321,113 @@ def _parse_lockfile_v9_test_impl(ctx): return unittest.end(env) +def _parse_lockfile_v9_injected_local_test_impl(ctx): + env = unittest.begin(ctx) + + parsed_json = pnpm.parse_pnpm_lock_json(json.encode({ + "lockfileVersion": "9.0", + "settings": { + "autoInstallPeers": True, + "excludeLinksFromLockfile": False, + }, + "importers": { + ".": {}, + "packages/a": { + "dependencies": { + "b": { + "specifier": "workspace:*", + "version": "file:packages/b(typescript@5.6.2)", + }, + }, + "dependenciesMeta": { + "b": { + "injected": True, + }, + }, + }, + "packages/b": { + "dependencies": { + "typescript": { + "specifier": "^5.6.2", + "version": "5.6.2", + }, + }, + }, + }, + "packages": { + "b@file:packages/b": { + "resolution": { + "directory": "packages/b", + "type": "directory", + }, + "name": "b", + "peerDependencies": { + "typescript": "^5.6.2", + }, + }, + "typescript@5.6.2": { + "resolution": { + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + }, + "engines": { + "node": ">=14.17", + }, + "hasBin": True, + }, + }, + "snapshots": { + "b@file:packages/b(typescript@5.6.2)": { + "id": "b@file:packages/b", + "dependencies": { + "typescript": "5.6.2", + }, + }, + "typescript@5.6.2": {}, + }, + })) + + # NOTE: unknown properties in >=v9 + v9_expected_packages = dict(expected_packages_injected) + for pkg_name in v9_expected_packages.keys(): + v9_expected_packages[pkg_name] = dict(v9_expected_packages[pkg_name]) + v9_expected_packages[pkg_name]["dev"] = None + v9_expected_packages[pkg_name]["requires_build"] = None + if pkg_name == "file:packages/b_typescript_5.6.2": + # This is incorrect in v6, but correct in v9 + # v6 is used as reference so we override for v9 here + v9_expected_packages[pkg_name]["friendly_version"] = "file:packages/b" + + expected = ( + expected_imports_injected, + v9_expected_packages, + {}, + 9.0, + None, + ) + + # Importers + asserts.equals(env, expected[0], parsed_json[0]) + + # Packages + # TODO v9 lockfile _should_ produce identical output but does not + # - package names are different + # Expected: file:packages/b_typescript_5.6.2 + # Actual: b@file:packages/b_typescript_5.6.2 + # - package id is missing + # Expected: file:packages/b + # Actual: None + # Update this assertion once v9 lockfile produces correct output + flawed_v9_expected_packages = dict(v9_expected_packages) + pkg_data = flawed_v9_expected_packages.pop("file:packages/b_typescript_5.6.2") + pkg_data = dict(pkg_data) + pkg_data["id"] = None + flawed_v9_expected_packages["b@file:packages/b_typescript_5.6.2"] = pkg_data + asserts.equals(env, flawed_v9_expected_packages, parsed_json[1]) + + asserts.equals(env, expected[2:], parsed_json[2:]) + + return unittest.end(env) + # buildifier: disable=function-docstring def _test_strip_peer_dep_or_patched_version(ctx): env = unittest.begin(ctx) @@ -198,6 +440,30 @@ def _test_strip_peer_dep_or_patched_version(ctx): asserts.equals(env, "21.1.0", pnpm_test.strip_v5_peer_dep_or_patched_version("21.1.0")) return unittest.end(env) +def _test_convert_pnpm_v6_v9_version_peer_dep(ctx): + env = unittest.begin(ctx) + asserts.equals( + env, + "1.2.3", + pnpm_test.convert_pnpm_v6_v9_version_peer_dep("1.2.3"), + ) + asserts.equals( + env, + "1.2.3_at_scope_peer_2.0.2", + pnpm_test.convert_pnpm_v6_v9_version_peer_dep("1.2.3(@scope/peer@2.0.2)"), + ) + asserts.equals( + env, + "1.2.3_2001974805", + pnpm_test.convert_pnpm_v6_v9_version_peer_dep("1.2.3(@scope/peer@2.0.2)(@scope/peer@4.5.6)"), + ) + asserts.equals( + env, + "4.5.6_o3deharooos255qt5xdujc3cuq", + pnpm_test.convert_pnpm_v6_v9_version_peer_dep("4.5.6(patch_hash=o3deharooos255qt5xdujc3cuq)"), + ) + return unittest.end(env) + # buildifier: disable=function-docstring def _test_version_supported(ctx): env = unittest.begin(ctx) @@ -224,6 +490,9 @@ c_test = unittest.make(_parse_lockfile_v6_test_impl, attrs = {}) d_test = unittest.make(_parse_lockfile_v9_test_impl, attrs = {}) e_test = unittest.make(_test_version_supported, attrs = {}) f_test = unittest.make(_test_strip_peer_dep_or_patched_version, attrs = {}) +g_test = unittest.make(_test_convert_pnpm_v6_v9_version_peer_dep, attrs = {}) +h_test = unittest.make(_parse_lockfile_v6_local_injected_test_impl, attrs = {}) +j_test = unittest.make(_parse_lockfile_v9_injected_local_test_impl, attrs = {}) TESTS = [ a_test, @@ -232,8 +501,19 @@ TESTS = [ d_test, e_test, f_test, + g_test, + h_test, + j_test, ] +# buildifier: disable=function-docstring def parse_pnpm_lock_tests(name): + tests = [] for index, test_rule in enumerate(TESTS): - test_rule(name = "{}_test_{}".format(name, index)) + test_name = "{}_test_{}".format(name, index) + test_rule(name = test_name) + tests.append(":" + test_name) + native.test_suite( + name = name, + tests = tests, + )