From 767b61dad57556afdac107ddf1691a36223b50f8 Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Mon, 18 Mar 2024 21:24:29 -0400 Subject: [PATCH 1/9] feat: add dump_syms --- .vscode/launch.json | 13 +++++++++++++ bin/command-line-definitions.ts | 10 +++++++++- bin/index.ts | 29 ++++++++++++++++++++++++++--- package.json | 1 + src/upload.ts | 12 +----------- 5 files changed, 50 insertions(+), 15 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index a0abed3..60384f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -56,6 +56,19 @@ ], "type": "node" }, + { + "name": "Start: dump_syms", + "request": "launch", + "runtimeArgs": [ + "run-script", + "start:dump_syms", + ], + "runtimeExecutable": "npm", + "skipFiles": [ + "/**" + ], + "type": "node" + }, { "name": "Test", "request": "launch", diff --git a/bin/command-line-definitions.ts b/bin/command-line-definitions.ts index 583a91b..ca5ac7a 100644 --- a/bin/command-line-definitions.ts +++ b/bin/command-line-definitions.ts @@ -85,6 +85,14 @@ export const argDefinitions: Array = [ typeLabel: '{underline string} (optional)', description: 'Path of the base directory used to search for symbol files. This value will be combined with the --files glob. Defaults to \'.\'', }, + { + name: 'dumpSyms', + alias: 'm', + type: Boolean, + defaultValue: false, + description: 'Use dump_syms to generate and upload sym files for specified binaries.', + typeLabel: '{underline boolean} (optional)', + } ]; export const usageDefinitions: Array
= [ @@ -134,7 +142,7 @@ function getPackageVersion(): string { } const packageJson = readFileSync(path, 'utf-8').toString(); - + try { return JSON.parse(packageJson).version; } catch { diff --git a/bin/index.ts b/bin/index.ts index c792622..31f54e9 100644 --- a/bin/index.ts +++ b/bin/index.ts @@ -2,11 +2,14 @@ import { ApiClient, BugSplatApiClient, OAuthClientCredentialsClient, VersionsApiClient } from '@bugsplat/js-api-client'; import commandLineArgs, { CommandLineOptions } from 'command-line-args'; import commandLineUsage from 'command-line-usage'; +import { glob } from 'glob'; +import { dumpSyms as nodeDumpSyms } from 'node-dump-syms'; +import { existsSync } from 'node:fs'; import { mkdir, readFile, stat } from 'node:fs/promises'; +import { basename, join } from 'node:path'; +import { safeRemoveTmp, tmpDir } from '../src/tmp'; import { uploadSymbolFiles } from '../src/upload'; import { CommandLineDefinition, argDefinitions, usageDefinitions } from './command-line-definitions'; -import { safeRemoveTmp, tmpDir } from '../src/tmp'; -import { existsSync } from 'node:fs'; (async () => { let { @@ -21,6 +24,7 @@ import { existsSync } from 'node:fs'; remove, files, directory, + dumpSyms } = await getCommandLineOptions(argDefinitions); if (help) { @@ -94,7 +98,26 @@ import { existsSync } from 'node:fs'; await mkdir(tmpDir); } - await uploadSymbolFiles(bugsplat, database, application, version, directory, files); + const globPattern = `${directory}/${files}`; + + let symbolFilePaths = await glob(globPattern); + + if (!symbolFilePaths.length) { + throw new Error(`Could not find any files to upload using glob ${globPattern}!`); + } + + console.log(`Found files:\n ${symbolFilePaths.join('\n')}`); + + if (dumpSyms) { + symbolFilePaths = symbolFilePaths.map(file => { + console.log(`Dumping syms for ${file}...`); + const symFile = join(tmpDir, `${basename(file)}.sym`); + nodeDumpSyms(file, symFile); + return symFile; + }); + } + + await uploadSymbolFiles(bugsplat, database, application, version, directory, symbolFilePaths); process.exit(0); })().catch((error) => { diff --git a/package.json b/package.json index a05e85f..181662c 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "start:dsym": "ts-node -r dotenv/config ./bin/index.ts -d ./spec/support -f \"*.dSYM\" -a BugSplatTester -v \"1.0 (1)\"", "start:xcarchive": "ts-node -r dotenv/config ./bin/index.ts -d ./spec/support -f \"*.xcarchive/**/*.dSYM\" -v \"4.5.6 (1)\"", "start:elf": "ts-node -r dotenv/config ./bin/index.ts -d ./spec -f \"**/*.elf\"", + "start:dump_syms": "ts-node -r dotenv/config ./bin/index.ts -d ./spec -f \"**/*.dSYM\" -m", "test": "ts-node node_modules/jasmine/bin/jasmine", "help": "ts-node ./bin/index.ts -h", "clean": "rimraf ./dist", diff --git a/src/upload.ts b/src/upload.ts index b199f9f..d8b5cf7 100644 --- a/src/upload.ts +++ b/src/upload.ts @@ -1,5 +1,4 @@ import { ApiClient, SymbolsApiClient, VersionsApiClient } from "@bugsplat/js-api-client"; -import { glob } from "glob"; import { basename, dirname, extname, join, relative } from "node:path"; import { pool } from "workerpool"; import { getDSymFileInfos } from './dsym'; @@ -11,16 +10,7 @@ import { createWorkersFromSymbolFiles } from './worker'; const workerPool = pool(join(__dirname, 'compression.js')); -export async function uploadSymbolFiles(bugsplat: ApiClient, database: string, application: string, version: string, directory: string, filesGlob: string) { - const globPattern = `${directory}/${filesGlob}`; - - const symbolFilePaths = await glob(globPattern); - - if (!symbolFilePaths.length) { - throw new Error(`Could not find any files to upload using glob ${globPattern}!`); - } - - console.log(`Found files:\n ${symbolFilePaths.join('\n')}`); +export async function uploadSymbolFiles(bugsplat: ApiClient, database: string, application: string, version: string, directory: string, symbolFilePaths: Array) { console.log(`About to upload symbols for ${database}-${application}-${version}...`); const symbolsApiClient = new SymbolsApiClient(bugsplat); From 3b27cea2f24b1a6b6c0c1efeba4e6b6b977a2258 Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Wed, 20 Mar 2024 12:18:24 -0400 Subject: [PATCH 2/9] fix: make node-dump-syms optional --- bin/index.ts | 18 +++++++++++------- package-lock.json | 16 ++++++++++++++++ package.json | 6 +++++- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/bin/index.ts b/bin/index.ts index 31f54e9..83ed820 100644 --- a/bin/index.ts +++ b/bin/index.ts @@ -3,7 +3,6 @@ import { ApiClient, BugSplatApiClient, OAuthClientCredentialsClient, VersionsApi import commandLineArgs, { CommandLineOptions } from 'command-line-args'; import commandLineUsage from 'command-line-usage'; import { glob } from 'glob'; -import { dumpSyms as nodeDumpSyms } from 'node-dump-syms'; import { existsSync } from 'node:fs'; import { mkdir, readFile, stat } from 'node:fs/promises'; import { basename, join } from 'node:path'; @@ -109,12 +108,17 @@ import { CommandLineDefinition, argDefinitions, usageDefinitions } from './comma console.log(`Found files:\n ${symbolFilePaths.join('\n')}`); if (dumpSyms) { - symbolFilePaths = symbolFilePaths.map(file => { - console.log(`Dumping syms for ${file}...`); - const symFile = join(tmpDir, `${basename(file)}.sym`); - nodeDumpSyms(file, symFile); - return symFile; - }); + try { + const nodeDumpSyms = (await import('node-dump-syms')).dumpSyms; + symbolFilePaths = symbolFilePaths.map(file => { + console.log(`Dumping syms for ${file}...`); + const symFile = join(tmpDir, `${basename(file)}.sym`); + nodeDumpSyms(file, symFile); + return symFile; + }); + } catch (error) { + console.log('node-dump-syms not found, skipping sym dumping...'); + } } await uploadSymbolFiles(bugsplat, database, application, version, directory, symbolFilePaths); diff --git a/package-lock.json b/package-lock.json index 52c6cc8..bab7065 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,9 @@ "rimraf": "^5.0.0", "ts-node": "^10.9.2", "typescript": "^5.3.3" + }, + "optionalDependencies": { + "node-dump-syms": "^3.0.6" } }, "node_modules/@75lb/deep-merge": { @@ -1997,6 +2000,13 @@ "node": ">=10" } }, + "node_modules/node-dump-syms": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/node-dump-syms/-/node-dump-syms-3.0.6.tgz", + "integrity": "sha512-zdxWP7niMjGrPbNPpOQK9ENECa1z9E3K8y3YUuyXc04pgtCVYRDsyJz3EnR6AehxGORVGcpH/eYaUbn7OUWCHQ==", + "hasInstallScript": true, + "optional": true + }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -4812,6 +4822,12 @@ "semver": "^7.3.5" } }, + "node-dump-syms": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/node-dump-syms/-/node-dump-syms-3.0.6.tgz", + "integrity": "sha512-zdxWP7niMjGrPbNPpOQK9ENECa1z9E3K8y3YUuyXc04pgtCVYRDsyJz3EnR6AehxGORVGcpH/eYaUbn7OUWCHQ==", + "optional": true + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", diff --git a/package.json b/package.json index 181662c..c19af10 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,8 @@ "outputPath": "./pkg", "compress": "GZip", "assets": [ - "package.json" + "package.json", + "node_modules/node-dump-syms/dist/native/index.addon" ] }, "devDependencies": { @@ -91,5 +92,8 @@ "promise-retry": "^2.0.1", "rxjs": "^7.8.1", "workerpool": "^6.5.1" + }, + "optionalDependencies": { + "node-dump-syms": "^3.0.6" } } From a2f3b0c8f065efd22c5f518fc27ea818d04e4c2d Mon Sep 17 00:00:00 2001 From: BugSplatService Date: Wed, 20 Mar 2024 16:25:14 +0000 Subject: [PATCH 3/9] fix: ignore missing optional dependency --- bin/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/index.ts b/bin/index.ts index 83ed820..2dfb825 100644 --- a/bin/index.ts +++ b/bin/index.ts @@ -109,6 +109,7 @@ import { CommandLineDefinition, argDefinitions, usageDefinitions } from './comma if (dumpSyms) { try { + // @ts-ignore: Cannot find module const nodeDumpSyms = (await import('node-dump-syms')).dumpSyms; symbolFilePaths = symbolFilePaths.map(file => { console.log(`Dumping syms for ${file}...`); From 45feb507312274a9ea40a4c26d9b0f9c88d7545b Mon Sep 17 00:00:00 2001 From: BugSplatService Date: Wed, 20 Mar 2024 16:30:51 +0000 Subject: [PATCH 4/9] fix: graceful dump_syms exit --- bin/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/index.ts b/bin/index.ts index 2dfb825..2f94a5d 100644 --- a/bin/index.ts +++ b/bin/index.ts @@ -117,8 +117,8 @@ import { CommandLineDefinition, argDefinitions, usageDefinitions } from './comma nodeDumpSyms(file, symFile); return symFile; }); - } catch (error) { - console.log('node-dump-syms not found, skipping sym dumping...'); + } catch (cause) { + throw new Error('Can\'t run dump_syms! Please ensure node-dump-syms is installed https://github.com/BugSplat-Git/node-dump-syms', { cause }); } } From d1e5e54079f1eaff5b369c1041be0b508bb98eb5 Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Wed, 20 Mar 2024 14:00:47 -0400 Subject: [PATCH 5/9] fix: remove tmp files --- bin/index.ts | 9 ++++----- src/worker.ts | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/bin/index.ts b/bin/index.ts index 2f94a5d..ba7d831 100644 --- a/bin/index.ts +++ b/bin/index.ts @@ -123,14 +123,13 @@ import { CommandLineDefinition, argDefinitions, usageDefinitions } from './comma } await uploadSymbolFiles(bugsplat, database, application, version, directory, symbolFilePaths); - + await safeRemoveTmp(); process.exit(0); -})().catch((error) => { +})().catch(async (error) => { console.error(error.message); - process.exit(1); -}).finally(async () => { await safeRemoveTmp(); -}) + process.exit(1); +}); async function createBugSplatClient({ user, diff --git a/src/worker.ts b/src/worker.ts index 0b0d21e..7762534 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -55,11 +55,11 @@ export class UploadWorker { let tmpFileName = ''; if (dbgId && !isZip) { - tmpFileName = join(tmpDir, `${fileName}.gz`); - let tmpSubdir = join(tmpDir, dirname(fileName)); + const tmpSubdir = join(tmpDir, dirname(fileName)); if (!existsSync(tmpSubdir)) { mkdirSync(tmpSubdir, { recursive: true }); } + tmpFileName = join(tmpSubdir, `${fileName}.gz`); client = this.symbolsClient; await this.pool.exec('createGzipFile', [path, tmpFileName]); } else if (!isZip) { From 7a52ab299ea1407411b69c66bde4fc3bf163f26a Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Wed, 20 Mar 2024 14:18:34 -0400 Subject: [PATCH 6/9] chore: pkg workflow --- .github/workflows/pkg.yaml | 39 ++++++++++++++++++++++++++++++++++++++ package.json | 14 ++++++-------- 2 files changed, 45 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/pkg.yaml diff --git a/.github/workflows/pkg.yaml b/.github/workflows/pkg.yaml new file mode 100644 index 0000000..d39ecb3 --- /dev/null +++ b/.github/workflows/pkg.yaml @@ -0,0 +1,39 @@ +name: Build and Upload Artifacts for Multiple OS + +on: [push] + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + run_script: pkg:linux + artifact_name: symbol-upload-linux + - os: macos-latest + run_script: pkg:macos + artifact_name: symbol-upload-macos + - os: windows-latest + run_script: pkg:win + artifact_name: symbol-upload-win + + steps: + - uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Install dependencies + run: npm ci + + - name: Run platform-specific script + run: npm run ${{ matrix.run_script }} # Executes the script based on the operating system + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.artifact_name }} + path: pkg/* # Uploads the artifact with a platform-specific name and path diff --git a/package.json b/package.json index c19af10..83be3c3 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,12 @@ "build": "tsc", "prerelease": "npm run build", "release": "npm publish --access public", - "prepkg": "npm run build", - "pkg": "npx pkg package.json", + "prepkg:linux": "npm run build", + "pkg:linux": "npx pkg package.json --targets node18-linux-x64 --output ./pkg/symbol-upload-linux", + "prepkg:macos": "npm run build", + "pkg:macos": "npx pkg package.json --targets node18-macos-x64 --output ./pkg/symbol-upload-macos", + "prepkg:windows": "npm run build", + "pkg:windows": "npx pkg package.json --targets node18-win-x64 --output ./pkg/symbol-upload-windows", "act": "act --secret-file .env" }, "repository": { @@ -48,12 +52,6 @@ "homepage": "https://github.com/BugSplat-Git/symbol-upload#readme", "pkg": { "scripts": "./dist/**/*.js", - "targets": [ - "node18-macos-x64", - "node18-linux-x64", - "node18-win-x64" - ], - "outputPath": "./pkg", "compress": "GZip", "assets": [ "package.json", From 5442ed58dafe4c172fc8cc27fefbbc21c0404168 Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Wed, 20 Mar 2024 14:32:36 -0400 Subject: [PATCH 7/9] chore: fix windows action --- .github/workflows/pkg.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pkg.yaml b/.github/workflows/pkg.yaml index d39ecb3..b0951ee 100644 --- a/.github/workflows/pkg.yaml +++ b/.github/workflows/pkg.yaml @@ -16,7 +16,7 @@ jobs: run_script: pkg:macos artifact_name: symbol-upload-macos - os: windows-latest - run_script: pkg:win + run_script: pkg:windows artifact_name: symbol-upload-win steps: From ad653651b0770c34709573c4333a08f0cdf2cd50 Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Wed, 20 Mar 2024 17:48:48 -0400 Subject: [PATCH 8/9] fix: log help and exit --- bin/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/index.ts b/bin/index.ts index ba7d831..ed5fd34 100644 --- a/bin/index.ts +++ b/bin/index.ts @@ -187,12 +187,12 @@ function logHelpAndExit() { function logMissingArgAndExit(arg: string): void { console.log(`\nMissing argument: -${arg}\n`); - process.exit(1); + logHelpAndExit() } function logMissingAuthAndExit(): void { console.log('\nInvalid authentication arguments: please provide either a user and password, or a clientId and clientSecret\n'); - process.exit(1); + logHelpAndExit() } function normalizeDirectory(directory: string): string { From 83fb909dfae9e95a1b83db9cffaf8b73e40e4482 Mon Sep 17 00:00:00 2001 From: Bobby Galli Date: Wed, 20 Mar 2024 18:05:12 -0400 Subject: [PATCH 9/9] chore: update actions --- .github/workflows/pkg.yaml | 20 +++++++++++++------- .github/workflows/pr.yaml | 6 +++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pkg.yaml b/.github/workflows/pkg.yaml index b0951ee..5f818bd 100644 --- a/.github/workflows/pkg.yaml +++ b/.github/workflows/pkg.yaml @@ -1,6 +1,10 @@ -name: Build and Upload Artifacts for Multiple OS +name: Package Builds + +on: + push: + tags: + - 'v*' -on: [push] jobs: build: @@ -20,20 +24,22 @@ jobs: artifact_name: symbol-upload-win steps: - - uses: actions/checkout@v4 + - name: ☑️ Checkout + uses: actions/checkout@v4 - - name: Setup Node.js + - name: ⚙️ Setup Node.js uses: actions/setup-node@v4 with: node-version: '18' - - name: Install dependencies + - name: 🏗️ Install Dependencies run: npm ci - - name: Run platform-specific script + - name: 📦 Run Pkg run: npm run ${{ matrix.run_script }} # Executes the script based on the operating system - - uses: actions/upload-artifact@v4 + - name: ⬆️ Upload Artifacts + uses: actions/upload-artifact@v4 with: name: ${{ matrix.artifact_name }} path: pkg/* # Uploads the artifact with a platform-specific name and path diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ebd6948..10b9ec8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -1,4 +1,4 @@ -name: Test My GitHub Action +name: Test GitHub Action on: pull_request: @@ -8,10 +8,10 @@ jobs: test-my-action: runs-on: ubuntu-latest steps: - - name: Check out code + - name: ☑️ Checkout uses: actions/checkout@v3 - - name: Symbols 📦 + - name: 📦 Symbols uses: ./ with: clientId: "${{ secrets.SYMBOL_UPLOAD_CLIENT_ID }}"