Start work on parsing yarn.lock file directly

.gitignore

@@ -5,3 +5,4 @@ result
 .pnp.loader.mjs
 **/.yarn/unplugged
 **/.yarn/install-state.gz
+tsconfig.tsbuildinfo

README.md

@@ -2,29 +2,34 @@
 
 Yet another way of packaging Node applications with Nix. Unlike alternatives, this plugin is built for performance (both speed and disk usage) and aims to be unique with the following goals:
 
-- NPM dependencies should be stored in the /nix/store individually rather than in a huge "node_modules" derivation. Zip files should be used where possible to work well with Yarn PNP
+- **Builds packages directly by reading yarn.lock**, no codegen\* or IFD needed
+- **NPM dependencies should be stored in the /nix/store individually** rather than in a huge "node_modules" derivation. Zip files should be used where possible to work well with Yarn PNP
 - Rebuilding when just changing source code should be fast as dependencies shouldn't be fetched
 - Adding a new dependency should just fetch that dependency rather than fetching or linking all node_modules again
 - Build native modules (e.g canvas) once, and once they are built they shouldn't be built again across packages
 - Unplugged/native modules should have their outputs hashed to try and enforce reproducibility
 - When using workspaces, adding a dependency in another package (modifying the yarn.lock file) in the workspace shouldn't cause a different package to have to be rebuilt
 - devDependencies shouldn't be included as references in the final runtime derivation, only dependencies
 
+\* Native / unplugged modules need to be flagged as such, along with their outputHashes for the built package, see examples in [test/flake.nix](./test/flake.nix). A Yarn plugin can be installed that will codegen a small manifest file that contains the list of native / unplugged modules.
+
 ## Usage
 
 Requires a Yarn version > 3 project using PnP linking (the default). Zero installs are not required, so it's recommended to just use the global cache when developing your project rather than storing dependencies in your repo.
 
+- Create a `flake.nix` if you haven't already for your project, add this repo (`yarnpnp2nix`) as an input (e.g `yarnpnp2nix.url = github:madjam002/yarnpnp2nix;`)
+
+- See [test/flake.nix](./test/flake.nix) for an example on how to create Nix derivations for Yarn packages using `mkYarnPackagesFromLockFile`.
+
+If you are using lots of native node modules, you may want to install the optional Yarn plugin which will codegen a small manifest that allows yarnpnp2nix to properly build native modules:
+
 - Install Yarn plugin in your project:
   ```
   yarn plugin import https://github.com/madjam002/yarnpnp2nix/raw/master/plugin/dist/plugin-yarnpnp2nix.js
   ```
 
 - Run `yarn` to make sure all packages are installed and to automatically generate a `yarn-manifest.nix` for your project.
 
-- Create a `flake.nix` if you haven't already for your project, add this repo (`yarnpnp2nix`) as an input (e.g `yarnpnp2nix.url = github:madjam002/yarnpnp2nix;`)
-
-- See [test/flake.nix](./test/flake.nix) for an example on how to create Nix derivations for Yarn packages using `mkYarnPackagesFromManifest`.
-
 ## Other notes
 
 Known caveats:
@@ -34,6 +39,11 @@ Possible future improvements:
 - When adding a Yarn package, copy it straight into the nix store rather than a Yarn cache in the users home directory
 - ...and run postinstall/install builds from within Nix by defaulting Yarn to --ignore-scripts
 
+## Troubleshooting
+
+### `hash mismatch in fixed-output derivation` when fetching dependencies
+
+If you get hash mismatches, the dependency probably is a native module or unplugged (has postinstall scripts). Install the Yarn plugin as directed above or manually set the `shouldBeUnplugged` attribute in `packageOverrides`.
 
 ## License
 

flake.nix

@@ -23,7 +23,9 @@
           yarn-plugin = pkgs.callPackage ./yarnPlugin.nix {};
         };
         lib = {
-          mkYarnPackagesFromManifest = (import ./lib/mkYarnPackage.nix { defaultPkgs = pkgs; lib = pkgs.lib; }).mkYarnPackagesFromManifest;
+          mkYarnPackagesFromLockFile = (import ./lib/mkYarnPackage.nix { defaultPkgs = pkgs; lib = pkgs.lib; }).mkYarnPackagesFromLockFile;
+          fromYAML = (import ./lib/fromYAML.nix { lib = pkgs.lib; });
+          parseYarnLock = (import ./lib/parseYarnLock.nix { lib = pkgs.lib; });
         };
         devShell = import ./shell.nix {
           inherit pkgs;

internal/.yarnrc.yml

@@ -0,0 +1,2 @@
+yarnPath: ../plugin/.yarn/releases/yarn-4.0.0-rc.25.cjs
+nodeLinker: node-modules

internal/bin/createLockFile.ts

@@ -0,0 +1,118 @@
+import * as fs from 'node:fs'
+import { Configuration, Locator, Package, Project, structUtils } from '@yarnpkg/core'
+import { ppath } from '@yarnpkg/fslib'
+import { cleanLocatorString } from '../lib'
+
+const packageRegistryDataPath = process.argv[3]
+const topLevelPackageLocatorString = process.argv[4]
+
+export default async function createLockFile() {
+  const configuration = await Configuration.find(ppath.cwd(), null);
+  const project = new Project(ppath.cwd(), { configuration })
+
+  await (project as any).setupResolutions() // HACK setupResolutions is private
+
+  const topLevelPackageLocator = structUtils.parseLocator(topLevelPackageLocatorString)
+
+  const packageRegistryData = JSON.parse(fs.readFileSync(packageRegistryDataPath, 'utf8'))
+
+  packageRegistryToProjectOriginalPackages(project, topLevelPackageLocator, packageRegistryData)
+
+  project.storedPackages = project.originalPackages
+
+  await project.persistLockfile()
+}
+
+function packageRegistryToProjectOriginalPackages(project: Project, topLevelPackageLocator: Locator, packageRegistryData: any) {
+  packageRegistryData["root-workspace-0b6124@workspace:."] = {
+    linkType: 'soft',
+    languageName: 'unknown',
+    packageDependencies: {
+      [structUtils.stringifyIdent(topLevelPackageLocator)]: structUtils.stringifyLocator(topLevelPackageLocator),
+    },
+  }
+
+  const packageRegistryDataEntries = Object.entries(packageRegistryData) as any
+
+  for (let [locatorString, pkg] of packageRegistryDataEntries) {
+    if (!pkg) continue
+
+    const isTopLevelPackage = locatorString === topLevelPackageLocatorString || locatorString === 'root-workspace-0b6124@workspace:.'
+
+    const dependencies = new Map()
+    const dependenciesMeta = new Map(Object.entries(pkg.dependenciesMeta ?? {}))
+    const peerDependencies = new Map()
+    const peerDependenciesMeta = isTopLevelPackage ? new Map() : new Map(Object.entries(pkg.peerDependenciesMeta ?? {}))
+    const bin = new Map(Object.entries(pkg.bin ?? {}))
+
+    locatorString = cleanLocatorString(locatorString)
+    const locator = structUtils.parseLocator(locatorString)
+
+    const ident = structUtils.makeIdent(locator.scope, locator.name)
+    const descriptor = structUtils.makeDescriptor(ident, locator.reference) // locators are also valid descriptors
+
+    pkg.locatorHash = locator.locatorHash
+    pkg.descriptorHash = descriptor.descriptorHash
+
+    if (!isTopLevelPackage) {
+      for (const dependencyName of Object.keys(pkg?.peerDependencies ?? {})) {
+        const ident = structUtils.parseIdent(dependencyName)
+        const descriptor = structUtils.makeDescriptor(ident, pkg.peerDependencies[dependencyName])
+        peerDependencies.set(ident.identHash, descriptor)
+      }
+    }
+
+    const origPackage: Package = {
+      ...locator,
+      languageName: pkg.languageName,
+      linkType: pkg.linkType.toUpperCase(),
+      conditions: null,
+      dependencies,
+      // TODO
+      // dependenciesMeta: dependenciesMeta as any,
+      dependenciesMeta: null as any,
+      bin: bin as any,
+      peerDependencies,
+      peerDependenciesMeta: peerDependenciesMeta as any,
+      version: null,
+    }
+    project.originalPackages.set(origPackage.locatorHash, origPackage)
+
+    // storedResolutions is a map of descriptorHash -> locatorHash
+    project.storedResolutions.set(descriptor.descriptorHash, origPackage.locatorHash)
+
+    // storedChecksums is a map of locatorHash -> checksum
+    if (pkg.checksum != null) project.storedChecksums.set(origPackage.locatorHash, '9/' + pkg.checksum)
+
+    project.storedDescriptors.set(descriptor.descriptorHash, descriptor)
+  }
+
+  for (const [locatorString, _package] of packageRegistryDataEntries) {
+    if (!_package) continue
+
+    const pkg = project.originalPackages.get(_package.locatorHash)
+    if (!pkg) continue
+
+    const pkgDependencies = _package.packageDependencies ?? {}
+
+    for (const dependencyName of Object.keys(pkgDependencies)) {
+      const depLocatorString = pkgDependencies[dependencyName]
+      const depPkg = packageRegistryData[depLocatorString]
+      if (depPkg?.descriptorHash != null) {
+        const depPkgDescriptor = project.storedDescriptors.get(depPkg.descriptorHash)
+        if (depPkgDescriptor != null) {
+          let descriptor = structUtils.makeDescriptor(structUtils.parseIdent(dependencyName), depPkgDescriptor.range)
+          const range = structUtils.parseRange(descriptor.range)
+
+          if (range.protocol === 'patch:') {
+            descriptor = structUtils.parseDescriptor(range.source!)
+          }
+
+          project.storedResolutions.set(descriptor.descriptorHash, depPkg.locatorHash)
+          project.storedDescriptors.set(descriptor.descriptorHash, descriptor)
+          pkg.dependencies.set(descriptor.identHash, descriptor)
+        }
+      }
+    }
+  }
+}

internal/bin/index.ts

@@ -0,0 +1,6 @@
+const commandMap = {
+  createLockFile: require('./createLockFile').default,
+  makePathWrappers: require('./makePathWrappers').default,
+}
+
+commandMap[process.argv[2]]()

internal/bin/makePathWrappers.ts

@@ -0,0 +1,36 @@
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+import { structUtils } from '@yarnpkg/core'
+import { xfs } from '@yarnpkg/fslib'
+import type * as PnpApi from 'pnpapi'
+import { readPackageJSON } from '../lib'
+
+const binWrappersOutDirectory = process.argv[3]
+const pnpOutDirectory = process.argv[4]
+
+export default async function makePathWrappers() {
+  const outDirectoryReal = fs.realpathSync(pnpOutDirectory)
+
+  const pnpApi: typeof PnpApi = require(path.join(outDirectoryReal, '.pnp.cjs'))
+  if (!pnpApi) throw new Error('Could not find pnp api')
+
+  const topLevelPackage = pnpApi.getPackageInformation(pnpApi.topLevel)
+
+  for (const [__, dep] of Object.entries(Array.from(topLevelPackage.packageDependencies))) {
+    const depLocator = (pnpApi as any).getLocator(dep[0], dep[1])
+    if (depLocator.reference == null) continue
+
+    const devirtualisedLocator = structUtils.ensureDevirtualizedLocator(depLocator)
+    const depPkg = pnpApi.getPackageInformation(depLocator)
+    const devirtualisedPkg = pnpApi.getPackageInformation(devirtualisedLocator)
+
+    const packageManifest = readPackageJSON(devirtualisedPkg)
+
+    for (const [bin, binScript] of Array.from(packageManifest.bin)) {
+      const resolvedBinPath = path.join(depPkg.packageLocation, binScript)
+      await xfs.writeFilePromise(path.join(binWrappersOutDirectory, bin) as any, `node ${resolvedBinPath} "$@"`, {
+        mode: 0o755,
+      })
+    }
+  }
+}

internal/build.js

@@ -0,0 +1,13 @@
+require('esbuild').build({
+  entryPoints: [
+    './bin/index',
+  ],
+  bundle: true,
+  outdir: 'dist/',
+  sourcemap: 'inline',
+  platform: 'node',
+  minify: true,
+  target: 'node18',
+  logLevel: 'warning',
+  treeShaking: true,
+})

internal/lib/index.ts

@@ -0,0 +1,27 @@
+import { structUtils, Manifest} from '@yarnpkg/core'
+import type * as PnpApi from 'pnpapi'
+import { ZipOpenFS } from '@yarnpkg/libzip'
+import { PosixFS } from '@yarnpkg/fslib'
+
+const libzip = require(`@yarnpkg/libzip`).getLibzipSync()
+
+const zipOpenFs = new ZipOpenFS({libzip});
+const crossFs = new PosixFS(zipOpenFs);
+
+export function cleanLocatorString(locatorString: string) {
+  const locator = structUtils.parseLocator(locatorString)
+  const range = structUtils.parseRange(locator.reference)
+
+  if (range.protocol === 'patch:') {
+    return structUtils.stringifyLocator({
+      ...locator,
+      reference: structUtils.makeRange({...range, params: null}),
+    })
+  }
+
+  return locatorString
+}
+
+export function readPackageJSON(packageInformation: PnpApi.PackageInformation) {
+  return Manifest.fromText(crossFs.readFileSync(packageInformation.packageLocation + 'package.json', 'utf8'))
+}

internal/package.json

@@ -0,0 +1,21 @@
+{
+  "private": true,
+  "scripts": {
+    "dev": "yarn nodemon -e ts build.js"
+  },
+  "devDependencies": {
+    "@types/node": "^18.11.9",
+    "@types/pnpapi": "^0.0.2",
+    "@yarnpkg/cli": "^4.0.0-rc.27",
+    "@yarnpkg/core": "^4.0.0-rc.27",
+    "@yarnpkg/fslib": "^3.0.0-rc.27",
+    "@yarnpkg/plugin-pnp": "^4.0.0-rc.27",
+    "@yarnpkg/pnp": "^4.0.0-rc.27",
+    "esbuild": "^0.15.13",
+    "nodemon": "^2.0.20",
+    "typescript": "^4.8.4"
+  },
+  "dependencies": {
+    "@yarnpkg/libzip": "^3.0.0-rc.27"
+  }
+}