diff --git a/.gitignore b/.gitignore index 215b8932c9ab..5c44baba89cb 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ zwave-js.tgz # ZWave cache */cache packages/*/cache +/test/config # Temporary (import) config files .tmp diff --git a/docs/api/driver.md b/docs/api/driver.md index 953bab82b6f3..56ade620d3bc 100644 --- a/docs/api/driver.md +++ b/docs/api/driver.md @@ -322,6 +322,8 @@ installConfigUpdate(): Promise Checks whether there is a compatible update for the currently installed config package and tries to install it. Returns `true` when an update was installed, `false` otherwise. +This requires an external configuration directory to be configured using the `deviceConfigExternalDir` driver option or the `ZWAVEJS_EXTERNAL_CONFIG` environment variable. + > [!NOTE] Although the updated config gets loaded after the update, bugfixes and changes to device configuration generally require either a driver restart or re-interview of the changed devices to take effect. ## Driver properties diff --git a/docs/getting-started/migrating/v14.md b/docs/getting-started/migrating/v14.md index dc9460ae4b6b..b0de3355f328 100644 --- a/docs/getting-started/migrating/v14.md +++ b/docs/getting-started/migrating/v14.md @@ -6,11 +6,9 @@ The `ZWaveHost` and `ZWaveApplicationHost` interfaces have been replaced by mult Furthermore, `Message` and `CommandClass` implementations are no longer bound to a specific host instance. Instead, their methods that need access to host functionality (like value DBs, home ID, device configuration, etc.) now receive a method-specific context object. Parsing of those instances no longer happens in the constructor, but in a separate `from` method. -In an attempt to make Z-Wave JS more portable, almost all usages of Node.js's `Buffer` class have been replaced with the native `Uint8Array`, or our new `Bytes` class that acts as a `Buffer` replacement. - Last but not least, the `npm` packages of Z-Wave JS are now hybrid ES Module & CommonJS. This means that consumers of Z-Wave JS can now benefit from the advantages of ES Modules, like tree-shaking or improved browser compatibility, without breaking compatibility with CommonJS-based projects. -All in all, this release contains a huge list of breaking changes, but most of those are limited low-level APIs. +All in all, this release contains a huge list of breaking changes, but most of those are limited to low-level or sparingly used APIs. ## Replaced Node.js `Buffer` with `Uint8Array` or portable `Bytes` class @@ -27,6 +25,12 @@ Both `Uint8Array` and `Bytes` can easily be converted to a `Buffer` instance if To test whether something is a `Uint8Array`, use the `isUint8Array` function exported from `node:util/types` (not portable) or `@zwave-js/shared` (portable). +## Configuration DB updates require an external config directory + +Previously, when a new version of the config DB should be installed, Z-Wave JS would attempt to update `@zwave-js/config` inside `node_modules`, unless it was installed in Docker, or an external configuration directory was configured. This exception is however true for a majority of our installations, so the added complexity of dealing with package managers was not worth it. + +Going forward, the `Driver.installConfigUpdate()` method will only install configuration updates if an external config directory is configured using either the `deviceConfigExternalDir` driver option or the `ZWAVEJS_EXTERNAL_CONFIG` environment variable. + ## Changed some default paths A few default paths have been changed to be relative to the cwd of the current process, rather than Z-Wave JS's source files: diff --git a/packages/config/src/ConfigManager.ts b/packages/config/src/ConfigManager.ts index dd8fde6dc2b5..04af72e7b56d 100644 --- a/packages/config/src/ConfigManager.ts +++ b/packages/config/src/ConfigManager.ts @@ -24,15 +24,17 @@ import { loadFulltextDeviceIndexInternal, } from "./devices/DeviceConfig.js"; import { + type SyncExternalConfigDirResult, configDir, - externalConfigDir, getDeviceEntryPredicate, + getExternalConfigDirEnvVariable, syncExternalConfigDir, } from "./utils.js"; export interface ConfigManagerOptions { logContainer?: ZWaveLogContainer; deviceConfigPriorityDir?: string; + deviceConfigExternalDir?: string; } export class ConfigManager { @@ -41,6 +43,8 @@ export class ConfigManager { options.logContainer ?? new ZWaveLogContainer({ enabled: false }), ); this.deviceConfigPriorityDir = options.deviceConfigPriorityDir; + this.deviceConfigExternalDir = options.deviceConfigExternalDir; + this._configVersion = PACKAGE_VERSION; } @@ -63,6 +67,12 @@ export class ConfigManager { } private deviceConfigPriorityDir: string | undefined; + private deviceConfigExternalDir: string | undefined; + public get externalConfigDir(): string | undefined { + return this.deviceConfigExternalDir + ?? getExternalConfigDirEnvVariable(); + } + private index: DeviceConfigIndex | undefined; private fulltextIndex: FulltextDeviceConfigIndex | undefined; @@ -74,11 +84,19 @@ export class ConfigManager { public async loadAll(): Promise { // If the environment option for an external config dir is set // try to sync it and then use it - const syncResult = await syncExternalConfigDir(this.logger); - if (syncResult.success) { + let syncResult: SyncExternalConfigDirResult | undefined; + const externalConfigDir = this.externalConfigDir; + if (externalConfigDir) { + syncResult = await syncExternalConfigDir( + externalConfigDir, + this.logger, + ); + } + + if (syncResult?.success) { this._useExternalConfig = true; this.logger.print( - `Using external configuration dir ${externalConfigDir()}`, + `Using external configuration dir ${externalConfigDir}`, ); this._configVersion = syncResult.version; } else { @@ -94,7 +112,7 @@ export class ConfigManager { public async loadManufacturers(): Promise { try { this._manufacturers = await loadManufacturersInternal( - this._useExternalConfig, + this._useExternalConfig && this.externalConfigDir || undefined, ); } catch (e) { // If the config file is missing or invalid, don't try to find it again @@ -163,7 +181,7 @@ export class ConfigManager { // The index of config files included in this package const embeddedIndex = await loadDeviceIndexInternal( this.logger, - this._useExternalConfig, + this._useExternalConfig && this.externalConfigDir || undefined, ); // A dynamic index of the user-defined priority device config files const priorityIndex: DeviceConfigIndex = []; @@ -251,7 +269,7 @@ export class ConfigManager { if (indexEntry) { const devicesDir = getDevicesPaths( - this._useExternalConfig ? externalConfigDir()! : configDir, + this._useExternalConfig && this.externalConfigDir || configDir, ).devicesDir; const filePath = path.isAbsolute(indexEntry.filename) ? indexEntry.filename diff --git a/packages/config/src/Manufacturers.ts b/packages/config/src/Manufacturers.ts index 5cb25c4ba992..f340ec7ab1dd 100644 --- a/packages/config/src/Manufacturers.ts +++ b/packages/config/src/Manufacturers.ts @@ -4,17 +4,17 @@ import { isObject } from "alcalzone-shared/typeguards/index.js"; import JSON5 from "json5"; import fs from "node:fs/promises"; import path from "node:path"; -import { configDir, externalConfigDir } from "./utils.js"; +import { configDir } from "./utils.js"; import { hexKeyRegex4Digits, throwInvalidConfig } from "./utils_safe.js"; export type ManufacturersMap = Map; /** @internal */ export async function loadManufacturersInternal( - externalConfig?: boolean, + externalConfigDir?: string, ): Promise { const configPath = path.join( - (externalConfig && externalConfigDir()) || configDir, + externalConfigDir || configDir, "manufacturers.json", ); diff --git a/packages/config/src/devices/DeviceConfig.ts b/packages/config/src/devices/DeviceConfig.ts index 93b346fc839c..297dc771021d 100644 --- a/packages/config/src/devices/DeviceConfig.ts +++ b/packages/config/src/devices/DeviceConfig.ts @@ -18,7 +18,7 @@ import path from "node:path"; import semver from "semver"; import { clearTemplateCache, readJsonWithTemplate } from "../JsonTemplate.js"; import type { ConfigLogger } from "../Logger.js"; -import { configDir, externalConfigDir } from "../utils.js"; +import { configDir } from "../utils.js"; import { hexKeyRegex4Digits, throwInvalidConfig } from "../utils_safe.js"; import { type AssociationConfig, @@ -305,10 +305,10 @@ export async function generatePriorityDeviceIndex( */ export async function loadDeviceIndexInternal( logger?: ConfigLogger, - externalConfig?: boolean, + externalConfigDir?: string, ): Promise { const { devicesDir, indexPath } = getDevicesPaths( - (externalConfig && externalConfigDir()) || configDir, + externalConfigDir || configDir, ); return loadDeviceIndexShared( diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 1264299118fb..fda4b49e65ca 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -10,4 +10,3 @@ export * from "./devices/DeviceMetadata.js"; export * from "./devices/EndpointConfig.js"; export * from "./devices/ParamInformation.js"; export * from "./devices/shared.js"; -export { externalConfigDir } from "./utils.js"; diff --git a/packages/config/src/utils.ts b/packages/config/src/utils.ts index c6ca5022adee..262ad62c9111 100644 --- a/packages/config/src/utils.ts +++ b/packages/config/src/utils.ts @@ -14,8 +14,9 @@ export const configDir = path.resolve( path.dirname(require.resolve("@zwave-js/config/package.json")), "config", ); + /** The (optional) absolute path of an external configuration directory */ -export function externalConfigDir(): string | undefined { +export function getExternalConfigDirEnvVariable(): string | undefined { return process.env.ZWAVEJS_EXTERNAL_CONFIG; } @@ -59,9 +60,9 @@ export type SyncExternalConfigDirResult = * Synchronizes or updates the external config directory and returns whether the directory is in a state that can be used */ export async function syncExternalConfigDir( + extConfigDir: string, logger: ConfigLogger, ): Promise { - const extConfigDir = externalConfigDir(); if (!extConfigDir) return { success: false }; // Make sure the config dir exists diff --git a/packages/zwave-js/package.json b/packages/zwave-js/package.json index aba397ae568f..8f22887fb946 100644 --- a/packages/zwave-js/package.json +++ b/packages/zwave-js/package.json @@ -98,7 +98,6 @@ }, "dependencies": { "@alcalzone/jsonl-db": "^3.1.1", - "@alcalzone/pak": "^0.11.0", "@homebridge/ciao": "^1.3.1", "@zwave-js/cc": "workspace:*", "@zwave-js/config": "workspace:*", diff --git a/packages/zwave-js/src/lib/driver/Driver.ts b/packages/zwave-js/src/lib/driver/Driver.ts index 152bdc6eac7a..0cf8f4d3281f 100644 --- a/packages/zwave-js/src/lib/driver/Driver.ts +++ b/packages/zwave-js/src/lib/driver/Driver.ts @@ -44,11 +44,7 @@ import { isMultiEncapsulatingCommandClass, isTransportServiceEncapsulation, } from "@zwave-js/cc"; -import { - ConfigManager, - type DeviceConfig, - externalConfigDir, -} from "@zwave-js/config"; +import { ConfigManager, type DeviceConfig } from "@zwave-js/config"; import { type CCId, CommandClasses, @@ -174,7 +170,6 @@ import { cloneDeep, createWrappingCounter, getErrorMessage, - isDocker, isUint8Array, mergeDeep, noop, @@ -260,11 +255,7 @@ import { type TransportServiceRXInterpreter, createTransportServiceRXMachine, } from "./TransportServiceMachine.js"; -import { - checkForConfigUpdates, - installConfigUpdate, - installConfigUpdateInDocker, -} from "./UpdateConfig.js"; +import { checkForConfigUpdates, installConfigUpdate } from "./UpdateConfig.js"; import { mergeUserAgent, userAgentComponentsToString } from "./UserAgent.js"; import type { EditableZWaveOptions, @@ -705,6 +696,8 @@ export class Driver extends TypedEventEmitter logContainer: this._logContainer, deviceConfigPriorityDir: this._options.storage.deviceConfigPriorityDir, + deviceConfigExternalDir: + this._options.storage.deviceConfigExternalDir, }); const self = this; @@ -7213,25 +7206,23 @@ ${handlers.length} left`, const newVersion = await this.checkForConfigUpdates(true); if (!newVersion) return false; - try { + const extConfigDir = this.configManager.externalConfigDir; + if (!this.configManager.useExternalConfig || !extConfigDir) { this.driverLog.print( - `Installing version ${newVersion} of configuration DB...`, + `Cannot update configuration DB update - external config directory is not set`, + "error", ); - // We have 3 variants of this. - const extConfigDir = externalConfigDir(); - if (this.configManager.useExternalConfig && extConfigDir) { - // 1. external config dir, leave node_modules alone - await installConfigUpdateInDocker(newVersion, { - cacheDir: this._options.storage.cacheDir, - configDir: extConfigDir, - }); - } else if (isDocker()) { - // 2. Docker, but no external config dir, extract into node_modules - await installConfigUpdateInDocker(newVersion); - } else { - // 3. normal environment, use npm/yarn to install a new version of @zwave-js/config - await installConfigUpdate(newVersion); - } + return false; + } + + this.driverLog.print( + `Installing version ${newVersion} of configuration DB...`, + ); + try { + await installConfigUpdate(newVersion, { + cacheDir: this.cacheDir, + configDir: extConfigDir, + }); } catch (e) { this.driverLog.print(getErrorMessage(e), "error"); return false; diff --git a/packages/zwave-js/src/lib/driver/UpdateConfig.ts b/packages/zwave-js/src/lib/driver/UpdateConfig.ts index 6982ac004d6a..dca2ac12b0fd 100644 --- a/packages/zwave-js/src/lib/driver/UpdateConfig.ts +++ b/packages/zwave-js/src/lib/driver/UpdateConfig.ts @@ -1,21 +1,13 @@ -import { type PackageManager, detectPackageManager } from "@alcalzone/pak"; import { ZWaveError, ZWaveErrorCodes } from "@zwave-js/core"; -import { - copyFilesRecursive, - getErrorMessage, - readJSON, -} from "@zwave-js/shared"; +import { copyFilesRecursive, getErrorMessage } from "@zwave-js/shared"; import { isObject } from "alcalzone-shared/typeguards/index.js"; import execa from "execa"; import fs from "node:fs/promises"; -import { createRequire } from "node:module"; import os from "node:os"; import * as path from "node:path"; import * as lockfile from "proper-lockfile"; import * as semver from "semver"; -const require = createRequire(import.meta.url); - /** * Checks whether there is a compatible update for the currently installed config package. * Returns the new version if there is an update, `undefined` otherwise. @@ -62,68 +54,12 @@ export async function checkForConfigUpdates( } /** - * Installs the update for @zwave-js/config with the given version. + * Downloads and installs the given configuration update. + * This only works if an external configuation directory is used. */ -export async function installConfigUpdate(newVersion: string): Promise { - // Check which package manager to use for the update - let pak: PackageManager; - try { - pak = await detectPackageManager({ - cwd: __dirname, - requireLockfile: false, - setCwdToPackageRoot: true, - }); - } catch { - throw new ZWaveError( - `Config update failed: No package manager detected or package.json not found!`, - ZWaveErrorCodes.Config_Update_PackageManagerNotFound, - ); - } - - const packageJsonPath = path.join(pak.cwd, "package.json"); - try { - await lockfile.lock(packageJsonPath, { - onCompromised: () => { - // do nothing - }, - }); - } catch { - throw new ZWaveError( - `Config update failed: Another installation is already in progress!`, - ZWaveErrorCodes.Config_Update_InstallFailed, - ); - } - - // And install it - const result = await pak.overrideDependencies({ - "@zwave-js/config": newVersion, - }); - - // Free the lock - try { - if (await lockfile.check(packageJsonPath)) { - await lockfile.unlock(packageJsonPath); - } - } catch { - // whatever - just don't crash - } - - if (result.success) return; - - throw new ZWaveError( - `Config update failed: Package manager exited with code ${result.exitCode} -${result.stderr}`, - ZWaveErrorCodes.Config_Update_InstallFailed, - ); -} - -/** - * Installs the update for @zwave-js/config with the given version. - * Version for Docker images that does not mess up the container if there's no yarn cache - */ -export async function installConfigUpdateInDocker( +export async function installConfigUpdate( newVersion: string, - external?: { + external: { configDir: string; cacheDir: string; }, @@ -150,32 +86,10 @@ export async function installConfigUpdateInDocker( ); } - let lockfilePath: string; - let lockfileOptions: lockfile.LockOptions; - if (external) { - lockfilePath = external.cacheDir; - lockfileOptions = { - lockfilePath: path.join(external.cacheDir, "config-update.lock"), - }; - } else { - // Acquire a lock so the installation doesn't run twice - let pak: PackageManager; - try { - pak = await detectPackageManager({ - cwd: __dirname, - requireLockfile: false, - setCwdToPackageRoot: true, - }); - } catch { - throw new ZWaveError( - `Config update failed: No package manager detected or package.json not found!`, - ZWaveErrorCodes.Config_Update_PackageManagerNotFound, - ); - } - - lockfilePath = path.join(pak.cwd, "package.json"); - lockfileOptions = {}; - } + const lockfilePath = external.cacheDir; + const lockfileOptions: lockfile.LockOptions = { + lockfilePath: path.join(external.cacheDir, "config-update.lock"), + }; try { await lockfile.lock(lockfilePath, { @@ -219,12 +133,8 @@ export async function installConfigUpdateInDocker( ); } + // Download the package tarball into the temporary directory const tarFilename = path.join(tmpDir, "zjs-config-update.tgz"); - const configModuleDir = path.dirname( - require.resolve("@zwave-js/config/package.json"), - ); - const extractedDir = path.join(tmpDir, "extracted"); - try { const handle = await fs.open(tarFilename, "w"); const fstream = handle.createWriteStream({ autoClose: true }); @@ -256,8 +166,9 @@ export async function installConfigUpdateInDocker( return path; } - // Extract it into a temporary folder, then overwrite the config node_modules with it + const extractedDir = path.join(tmpDir, "extracted"); try { + // Extract the tarball in the temporary folder await fs.rm(extractedDir, { recursive: true, force: true }); await fs.mkdir(extractedDir, { recursive: true }); await execa("tar", [ @@ -267,25 +178,20 @@ export async function installConfigUpdateInDocker( "-C", normalizeToUnixStyle(extractedDir), ]); - // How we install now depends on whether we're installing into the external config dir. - // If we are, we just need to copy the `devices` subdirectory. If not, copy the entire extracted dir - if (external) { - await fs.rm(external.configDir, { recursive: true, force: true }); - await fs.mkdir(external.configDir, { recursive: true }); - await copyFilesRecursive( - path.join(extractedDir, "config"), - external.configDir, - (src) => src.endsWith(".json"), - ); - const externalVersionFilename = path.join( - external.configDir, - "version", - ); - await fs.writeFile(externalVersionFilename, newVersion, "utf8"); - } else { - await fs.rm(configModuleDir, { recursive: true, force: true }); - await fs.rename(extractedDir, configModuleDir); - } + + // then overwrite the files in the external config directory + await fs.rm(external.configDir, { recursive: true, force: true }); + await fs.mkdir(external.configDir, { recursive: true }); + await copyFilesRecursive( + path.join(extractedDir, "config"), + external.configDir, + (src) => src.endsWith(".json"), + ); + const externalVersionFilename = path.join( + external.configDir, + "version", + ); + await fs.writeFile(externalVersionFilename, newVersion, "utf8"); } catch { await freeLock(); throw new ZWaveError( @@ -294,18 +200,6 @@ export async function installConfigUpdateInDocker( ); } - // Try to update our own package.json if we're working with the internal structure - if (!external) { - try { - const packageJsonPath = require.resolve("zwave-js/package.json"); - const json = await readJSON(packageJsonPath); - json.dependencies["@zwave-js/config"] = newVersion; - await fs.writeFile(packageJsonPath, JSON.stringify(json, null, 2)); - } catch { - // ignore - } - } - // Clean up the temp dir and ignore errors void fs.rm(tmpDir, { recursive: true, force: true }).catch(() => { // ignore diff --git a/packages/zwave-js/src/lib/driver/ZWaveOptions.ts b/packages/zwave-js/src/lib/driver/ZWaveOptions.ts index 268e6acbc5f5..e52f2473a264 100644 --- a/packages/zwave-js/src/lib/driver/ZWaveOptions.ts +++ b/packages/zwave-js/src/lib/driver/ZWaveOptions.ts @@ -131,6 +131,15 @@ export interface ZWaveOptions extends ZWaveHostOptions { * Can also be set with the ZWAVEJS_LOCK_DIRECTORY env variable. */ lockDir?: string; + + /** + * Allows you to specify a directory where the embedded device configuration files are stored. + * When set, the configuration files can automatically be updated using `Driver.installConfigUpdate()` + * without having to update the npm packages. + * Can also be set using the ZWAVEJS_EXTERNAL_CONFIG env variable. + */ + deviceConfigExternalDir?: string; + /** * Allows you to specify a directory where device configuration files can be loaded from with higher priority than the included ones. * This directory does not get indexed and should be used sparingly, e.g. for testing. diff --git a/test/run.ts b/test/run.ts index ff446ced1b92..66200b968cdb 100644 --- a/test/run.ts +++ b/test/run.ts @@ -64,6 +64,7 @@ const driver = new Driver(port, { storage: { cacheDir: path.join(__dirname, "cache"), lockDir: path.join(__dirname, "cache/locks"), + deviceConfigExternalDir: path.join(__dirname, "config"), }, allowBootloaderOnly: true, }) diff --git a/yarn.lock b/yarn.lock index 3255fee52019..bd054531817b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10054,7 +10054,6 @@ __metadata: dependencies: "@alcalzone/esm2cjs": "npm:^1.2.3" "@alcalzone/jsonl-db": "npm:^3.1.1" - "@alcalzone/pak": "npm:^0.11.0" "@homebridge/ciao": "npm:^1.3.1" "@microsoft/api-extractor": "npm:^7.47.9" "@types/node": "npm:^18.19.63"