From e45ae8de58837a52ca6f8b0503152e2f87f55356 Mon Sep 17 00:00:00 2001 From: Anton Golub Date: Thu, 20 Jun 2024 21:38:18 +0300 Subject: [PATCH] build: split vendor chunk (#856) * build: split vendor chunk * build: enhance dts gen * ci: use Node.js 22 as builder --- .github/workflows/dev-publish.yml | 2 +- .github/workflows/npm-publish.yml | 2 +- .github/workflows/test.yml | 12 ++-- package.json | 4 +- scripts/build-dts.mjs | 106 +++++++++++++++++++----------- scripts/build-js.mjs | 2 +- src/core.ts | 2 +- src/util.ts | 35 +++++----- src/vendor-core.ts | 22 +++++++ src/vendor-extra.ts | 68 +++++++++++++++++++ src/vendor.ts | 65 +----------------- test/util.test.js | 14 ++-- 12 files changed, 197 insertions(+), 137 deletions(-) create mode 100644 src/vendor-core.ts create mode 100644 src/vendor-extra.ts diff --git a/.github/workflows/dev-publish.yml b/.github/workflows/dev-publish.yml index 7676bc8f98..f2e64bfc77 100644 --- a/.github/workflows/dev-publish.yml +++ b/.github/workflows/dev-publish.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 - run: npm ci - run: npm test env: diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index c04fd9f9ff..6a0dc735d9 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 - run: npm ci - run: npm test env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10a9e92d0f..13821c7ac1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,12 +7,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Use Node.js 20.x + - name: Use Node.js 22.x uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: 22.x - run: npm ci - - run: npm run build + - run: | + npm run build + cd build && ls -l - uses: actions/upload-artifact@v4 with: name: build @@ -126,10 +128,10 @@ jobs: ts: [4, 5] steps: - uses: actions/checkout@v4 - - name: Use Node.js 20.x + - name: Use Node.js 22.x uses: actions/setup-node@v4 with: - node-version: 20.x + node-version: 22.x - name: Install deps run: npm ci - name: Install TypeScript ${{ matrix.ts }} diff --git a/package.json b/package.json index 12ec897b6e..9f7315c0ed 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "build": "npm run build:js && npm run build:dts", "build:check": "tsc", "build:js": "node scripts/build-js.mjs --format=cjs --hybrid --entry=src/*.ts && npm run build:vendor", - "build:vendor": "node scripts/build-js.mjs --format=cjs --entry=src/vendor.ts --bundle=all", + "build:vendor": "node scripts/build-js.mjs --format=cjs --entry=src/vendor-*.ts --bundle=all", "build:dts": "tsc --project tsconfig.prod.json && node scripts/build-dts.mjs", "pretest": "npm run build", "test": "npm run test:unit && npm run test:types && npm run test:license", @@ -81,7 +81,7 @@ "test:smoke:cjs": "node ./test/smoke/node.test.cjs", "test:smoke:mjs": "node ./test/smoke/node.test.mjs", "test:smoke:deno": "deno test ./test/smoke/deno.test.js --allow-read --allow-sys --allow-env --allow-run", - "coverage": "c8 -x build/deno.js -x build/vendor.cjs -x build/esblib.cjs -x 'test/**' -x scripts --check-coverage npm test", + "coverage": "c8 -x build/deno.js -x build/vendor-extra.cjs -x build/vendor-core.cjs -x build/esblib.cjs -x 'test/**' -x scripts --check-coverage npm test", "version": "cat package.json | fx .version" }, "optionalDependencies": { diff --git a/scripts/build-dts.mjs b/scripts/build-dts.mjs index 4691aec0da..d79d786aab 100644 --- a/scripts/build-dts.mjs +++ b/scripts/build-dts.mjs @@ -18,45 +18,62 @@ import fs from 'fs/promises' import { generateDtsBundle } from 'dts-bundle-generator' import glob from 'fast-glob' -const entry = { - filePath: './src/vendor.ts', - outFile: './build/vendor.d.ts', - libraries: { - allowedTypesLibraries: ['node'], // args['external-types'], - inlinedLibraries: [ - '@nodelib/fs.stat', - '@nodelib/fs.scandir', - '@nodelib/fs.walk', - 'fast-glob', - '@types/jsonfile', - 'node-fetch-native', - 'chalk', - 'globby', - '@types/minimist', - '@types/which', - 'zurk', - '@webpod/ps', - '@webpod/ingrid', - 'depseek', - ], // args['external-inlines'], +const output = { + inlineDeclareExternals: true, + inlineDeclareGlobals: true, + sortNodes: false, + exportReferencedTypes: false, //args['export-referenced-types'], +} +const entries = [ + { + filePath: './src/vendor-extra.ts', + outFile: './build/vendor-extra.d.ts', + libraries: { + allowedTypesLibraries: ['node'], // args['external-types'], + inlinedLibraries: [ + '@nodelib/fs.stat', + '@nodelib/fs.scandir', + '@nodelib/fs.walk', + 'fast-glob', + '@types/jsonfile', + 'node-fetch-native', + // 'chalk', + 'globby', + // '@types/minimist', + // '@types/which', + // 'zurk', + // '@webpod/ps', + '@webpod/ingrid', + 'depseek', + ], // args['external-inlines'], + }, + output, }, - output: { - inlineDeclareExternals: true, - inlineDeclareGlobals: true, - sortNodes: false, - exportReferencedTypes: false, //args['export-referenced-types'], + { + filePath: './src/vendor-core.ts', + outFile: './build/vendor-core.d.ts', + libraries: { + allowedTypesLibraries: ['node'], // args['external-types'], + inlinedLibraries: [ + '@types/which', + '@webpod/ps', + '@webpod/ingrid', + 'chalk', + 'zurk', + ], // args['external-inlines'], + }, + output, }, -} +] const compilationOptions = { preferredConfigPath: './tsconfig.prod.json', // args.project, followSymlinks: true, } -let [result] = generateDtsBundle([entry], compilationOptions) - -// generateDtsBundle cannot handle the circular refs on types inlining, so we need to help it manually: -/* +const results = generateDtsBundle(entries, compilationOptions) + // generateDtsBundle cannot handle the circular refs on types inlining, so we need to help it manually: + /* build/vendor.d.ts(163,7): error TS2456: Type alias 'Options' circularly references itself. build/vendor.d.ts(164,7): error TS2456: Type alias 'Entry' circularly references itself. build/vendor.d.ts(165,7): error TS2456: Type alias 'Task' circularly references itself. @@ -65,17 +82,28 @@ build/vendor.d.ts(167,7): error TS2456: Type alias 'FileSystemAdapter' circularl build/vendor.d.ts(197,48): error TS2694: Namespace 'FastGlob' has no exported member 'FastGlobOptions */ -result = result - .replace('type Options = Options;', 'export {Options};') - .replace('type Task = Task;', 'export {Task};') - .replace('type Pattern = Pattern;', 'export {Pattern};') - .replace('FastGlob.FastGlobOptions', 'FastGlob.Options') - .replace('type Entry =', 'export type Entry =') + .map((r) => + r + .replace('type Options = Options;', 'export {Options};') + .replace('type Task = Task;', 'export {Task};') + .replace('type Pattern = Pattern;', 'export {Pattern};') + .replace('FastGlob.FastGlobOptions', 'FastGlob.Options') + .replace('type Entry =', 'export type Entry =') + ) -await fs.writeFile(entry.outFile, result, 'utf8') +for (const i in results) { + const entry = entries[i] + const result = results[i] + + await fs.writeFile(entry.outFile, result, 'utf8') +} // Replaces redundant triple-slash directives -for (const dts of await glob(['build/**/*.d.ts', '!build/vendor.d.ts'])) { +for (const dts of await glob([ + 'build/**/*.d.ts', + '!build/vendor.d.ts', + '!build/core-vendor.d.ts', +])) { const contents = (await fs.readFile(dts, 'utf8')) .split('\n') .filter((line) => !line.startsWith('/// path.relative(cwd, path.resolve(cwd, p))) diff --git a/src/core.ts b/src/core.ts index 76fd9963e7..7352777b00 100644 --- a/src/core.ts +++ b/src/core.ts @@ -34,7 +34,7 @@ import { type RequestInfo, type RequestInit, type TSpawnStore, -} from './vendor.js' +} from './vendor-core.js' import { type Duration, errnoMessage, diff --git a/src/util.ts b/src/util.ts index 8bc0212658..19f86dc5c8 100644 --- a/src/util.ts +++ b/src/util.ts @@ -14,7 +14,8 @@ import os from 'node:os' import path from 'node:path' -import { chalk, parseLine, fs } from './vendor.js' +import fs from 'node:fs' +import { chalk } from './vendor-core.js' export function tempdir(prefix = `zx-${randomId()}`) { const dirpath = path.join(os.tmpdir(), prefix) @@ -68,22 +69,22 @@ export function preferNmBin( } } -export function normalizeMultilinePieces( - pieces: TemplateStringsArray -): TemplateStringsArray { - return Object.assign( - pieces.map((p, i) => - p.trim() - ? pad(p[0]) + - parseLine(p) - .words.map(({ w }) => (w === '\\' ? '' : w.trim())) - .join(' ') + - pad(p[p.length - 1]) - : pieces[i] - ), - { raw: pieces.raw } - ) -} +// export function normalizeMultilinePieces( +// pieces: TemplateStringsArray +// ): TemplateStringsArray { +// return Object.assign( +// pieces.map((p, i) => +// p.trim() +// ? pad(p[0]) + +// parseLine(p) +// .words.map(({ w }) => (w === '\\' ? '' : w.trim())) +// .join(' ') + +// pad(p[p.length - 1]) +// : pieces[i] +// ), +// { raw: pieces.raw } +// ) +// } export function quote(arg: string) { if (/^[a-z0-9/_.\-@:=]+$/i.test(arg) || arg === '') { diff --git a/src/vendor-core.ts b/src/vendor-core.ts new file mode 100644 index 0000000000..620a6770ac --- /dev/null +++ b/src/vendor-core.ts @@ -0,0 +1,22 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export { exec, buildCmd, type TSpawnStore } from 'zurk/spawn' + +export type RequestInfo = Parameters[0] +export type RequestInit = Parameters[1] + +export { default as chalk, type ChalkInstance } from 'chalk' +export { default as which } from 'which' +export { default as ps } from '@webpod/ps' diff --git a/src/vendor-extra.ts b/src/vendor-extra.ts new file mode 100644 index 0000000000..573c5165ff --- /dev/null +++ b/src/vendor-extra.ts @@ -0,0 +1,68 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { URL } from 'node:url' +import { + convertPathToPattern, + globby, + globbySync, + globbyStream, + generateGlobTasksSync, + generateGlobTasks, + isGitIgnoredSync, + isGitIgnored, + isDynamicPattern, + type Options as GlobbyOptions, +} from 'globby' +import * as yaml from 'yaml' +import * as _fs from 'fs-extra' +import _createRequire from 'create-require' +import { type fetch, AbortController } from 'node-fetch-native' + +export { fetch as nodeFetch } from 'node-fetch-native' + +global.AbortController = global.AbortController || AbortController + +export const createRequire = _createRequire as unknown as ( + filename: string | URL +) => NodeRequire + +export const globbyModule = { + convertPathToPattern, + globby, + globbySync, + globbyStream, + generateGlobTasksSync, + generateGlobTasks, + isGitIgnoredSync, + isGitIgnored, + isDynamicPattern, +} + +export const glob = Object.assign(function globby( + patterns: string | readonly string[], + options?: GlobbyOptions +) { + return globbyModule.globby(patterns, options) +}, globbyModule) as (typeof globbyModule)['globby'] & typeof globbyModule + +export const YAML: { + parse(text: string): any + stringify(object: any): string +} = yaml + +export const fs: typeof import('fs-extra') = _fs + +export { depseekSync as depseek } from 'depseek' +export { default as minimist } from 'minimist' diff --git a/src/vendor.ts b/src/vendor.ts index ea98170eec..f5baf191c8 100644 --- a/src/vendor.ts +++ b/src/vendor.ts @@ -12,66 +12,5 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { URL } from 'node:url' -import { - convertPathToPattern, - globby, - globbySync, - globbyStream, - generateGlobTasksSync, - generateGlobTasks, - isGitIgnoredSync, - isGitIgnored, - isDynamicPattern, - type Options as GlobbyOptions, -} from 'globby' -import * as yaml from 'yaml' -import * as _fs from 'fs-extra' -import { type fetch, AbortController } from 'node-fetch-native' - -export { exec, buildCmd, type TSpawnStore } from 'zurk/spawn' - -import _createRequire from 'create-require' - -export const createRequire = _createRequire as unknown as ( - filename: string | URL -) => NodeRequire - -global.AbortController = global.AbortController || AbortController - -export { fetch as nodeFetch } from 'node-fetch-native' -export type RequestInfo = Parameters[0] -export type RequestInit = Parameters[1] - -export const globbyModule = { - convertPathToPattern, - globby, - globbySync, - globbyStream, - generateGlobTasksSync, - generateGlobTasks, - isGitIgnoredSync, - isGitIgnored, - isDynamicPattern, -} - -export const glob = Object.assign(function globby( - patterns: string | readonly string[], - options?: GlobbyOptions -) { - return globbyModule.globby(patterns, options) -}, globbyModule) as (typeof globbyModule)['globby'] & typeof globbyModule - -export const YAML: { - parse(text: string): any - stringify(object: any): string -} = yaml - -export const fs: typeof import('fs-extra') = _fs - -export { depseekSync as depseek } from 'depseek' -export { default as chalk, type ChalkInstance } from 'chalk' -export { default as which } from 'which' -export { default as minimist } from 'minimist' -export { default as ps } from '@webpod/ps' -export { parseLine } from '@webpod/ingrid' +export * from './vendor-core.js' +export * from './vendor-extra.js' diff --git a/test/util.test.js b/test/util.test.js index 063e44d5b4..c3b77fbaff 100644 --- a/test/util.test.js +++ b/test/util.test.js @@ -25,7 +25,7 @@ import { quote, quotePowerShell, randomId, - normalizeMultilinePieces, + // normalizeMultilinePieces, getCallerLocationFromString, tempdir, tempfile, @@ -98,12 +98,12 @@ describe('util', () => { ) }) - test('normalizeMultilinePieces()', () => { - assert.equal( - normalizeMultilinePieces([' a ', 'b c d', ' e']).join(','), - ' a ,b c d, e' - ) - }) + // test('normalizeMultilinePieces()', () => { + // assert.equal( + // normalizeMultilinePieces([' a ', 'b c d', ' e']).join(','), + // ' a ,b c d, e' + // ) + // }) }) test('getCallerLocation: empty', () => {