refactor: replace Node.js Buffer with Uint8Array or portable replacem…

…ent (#7332)

docs/getting-started/migrating/v14.md

@@ -6,7 +6,24 @@ The `ZWaveHost` and `ZWaveApplicationHost` interfaces have been replaced by mult
 
 Furthermore, `Message` and `CommandClass` implementations are no longer bound to a specific host instance. Instead, their methods that need access to host functionality (like value DBs, home ID, device configuration, etc.) now receive a method-specific context object. Parsing of those instances no longer happens in the constructor, but in a separate `from` method.
 
-All in all, this release contains a huge list of breaking changes, but most of those should not affect any application, unless very low-level APIs are frequently used.
+In an attempt to make Z-Wave JS more portable, almost all usages of Node.js's `Buffer` class have been replaced with the native `Uint8Array`, or our new `Bytes` class that acts as a `Buffer` replacement.
+
+All in all, this release contains a huge list of breaking changes, but most of those are limited low-level APIs.
+
+## Replaced Node.js `Buffer` with `Uint8Array` or portable `Bytes` class
+
+Since the beginning of Z-Wave JS, we've been using Node.js's `Buffer` class to manipulate binary data. This works fine, but is not portable, and prevents us from exploring compatibility with other runtimes, or even doing things in the browser, e.g. flashing controllers or modifying NVM contents.
+Following [Sindre Sorhus's example](https://sindresorhus.com/blog/goodbye-nodejs-buffer), the use of `Buffer`s was replaced with `Uint8Array`s where applicable.
+
+In input positions where Z-Wave JS previously accepted `Buffer`s, this change is backwards compatible, as `Buffer` is a subclass of `Uint8Array`. Applications can simply continue passing `Buffer` instances to Z-Wave JS.
+
+In output positions however, this is a breaking change. Where applications are affected by this, check if `Buffer` methods like `readUInt32BE` etc. are actually needed, or if changing the expected type to `Uint8Array` would be enough. This is usually the case when just passing the binary data around, or accessing its content by index.
+
+In some cases, Z-Wave JS now uses an almost drop-in replacement for Node.js's `Buffer`, the new `Bytes` class exported from `@zwave-js/shared`. This is a portable subclass of `Uint8Array` with some additions to make its API more (but not 100%) compatible with `Buffer`. It supports most of the `Buffer` functionality like `from()`, `concat()`, `toString()` (with limitations), `read/write[U]Int...LE/BE`. This should generally not leak into the public API, except for some rare edge cases.
+
+Both `Uint8Array` and `Bytes` can easily be converted to a `Buffer` instance if absolutely necessary by using `Buffer.from(...)`.
+
+To test whether something is a `Uint8Array`, use the `isUint8Array` function exported from `node:util/types` (not portable) or `@zwave-js/shared` (portable).
 
 ## Decoupled Serial API messages from host instances, split constructors and parsing
 

eslint.config.mjs

@@ -155,6 +155,17 @@ export default tseslint.config(
 			"unicorn/prefer-string-slice": "error",
 			"unicorn/prefer-string-starts-ends-with": "error",
 			"unicorn/prefer-string-replace-all": "error",
+
+			// Prefer our own Buffer implementation (compatible with native Uint8array)
+			// See https://sindresorhus.com/blog/goodbye-nodejs-buffer for the reason behind this
+			"no-restricted-globals": [
+				"error",
+				{
+					name: "Buffer",
+					message:
+						"Use Uint8Array or the Bytes implementation from @zwave-js/shared instead.",
+				},
+			],
 		},
 	},
 	// Disable unnecessarily strict rules for test files

packages/cc/src/cc/AlarmSensorCC.ts

@@ -17,6 +17,7 @@ import type {
 	CCParsingContext,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import { getEnumMemberName, isEnumMember, pick } from "@zwave-js/shared/safe";
 import { validateArgs } from "@zwave-js/transformers";
 import { CCAPI, PhysicalCCAPI } from "../lib/API";
@@ -449,8 +450,8 @@ export class AlarmSensorCCGet extends AlarmSensorCC {
 
 	public sensorType: AlarmSensorType;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.sensorType]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.sensorType]);
 		return super.serialize(ctx);
 	}
 

packages/cc/src/cc/AssociationCC.ts

@@ -20,6 +20,7 @@ import type {
 	GetDeviceConfig,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import { validateArgs } from "@zwave-js/transformers";
 import { distinct } from "alcalzone-shared/arrays";
 import { CCAPI, PhysicalCCAPI } from "../lib/API";
@@ -539,8 +540,8 @@ export class AssociationCCSet extends AssociationCC {
 	public groupId: number;
 	public nodeIds: number[];
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.groupId, ...this.nodeIds]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.groupId, ...this.nodeIds]);
 		return super.serialize(ctx);
 	}
 
@@ -598,8 +599,8 @@ export class AssociationCCRemove extends AssociationCC {
 	public groupId?: number;
 	public nodeIds?: number[];
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([
 			this.groupId || 0,
 			...(this.nodeIds || []),
 		]);
@@ -692,8 +693,8 @@ export class AssociationCCReport extends AssociationCC {
 			.reduce((prev, cur) => prev.concat(...cur), []);
 	}
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([
 			this.groupId,
 			this.maxNodes,
 			this.reportsToFollow,
@@ -748,8 +749,8 @@ export class AssociationCCGet extends AssociationCC {
 
 	public groupId: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.groupId]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.groupId]);
 		return super.serialize(ctx);
 	}
 
@@ -792,8 +793,8 @@ export class AssociationCCSupportedGroupingsReport extends AssociationCC {
 	@ccValue(AssociationCCValues.groupCount)
 	public groupCount: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.groupCount]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.groupCount]);
 		return super.serialize(ctx);
 	}
 
@@ -839,8 +840,8 @@ export class AssociationCCSpecificGroupReport extends AssociationCC {
 
 	public group: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.group]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.group]);
 		return super.serialize(ctx);
 	}
 

packages/cc/src/cc/AssociationGroupInfoCC.ts

@@ -17,6 +17,7 @@ import type {
 	CCParsingContext,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import { cpp2js, getEnumMemberName, num2hex } from "@zwave-js/shared/safe";
 import { validateArgs } from "@zwave-js/transformers";
 import { CCAPI, PhysicalCCAPI } from "../lib/API";
@@ -520,10 +521,10 @@ export class AssociationGroupInfoCCNameReport extends AssociationGroupInfoCC {
 		return true;
 	}
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.concat([
-			Buffer.from([this.groupId, this.name.length]),
-			Buffer.from(this.name, "utf8"),
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.concat([
+			Bytes.from([this.groupId, this.name.length]),
+			Bytes.from(this.name, "utf8"),
 		]);
 		return super.serialize(ctx);
 	}
@@ -569,8 +570,8 @@ export class AssociationGroupInfoCCNameGet extends AssociationGroupInfoCC {
 
 	public groupId: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.groupId]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.groupId]);
 		return super.serialize(ctx);
 	}
 
@@ -663,8 +664,8 @@ export class AssociationGroupInfoCCInfoReport extends AssociationGroupInfoCC {
 		return true;
 	}
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.alloc(1 + this.groups.length * 7, 0);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.alloc(1 + this.groups.length * 7, 0);
 
 		this.payload[0] = (this.isListMode ? 0b1000_0000 : 0)
 			| (this.hasDynamicInfo ? 0b0100_0000 : 0)
@@ -673,7 +674,7 @@ export class AssociationGroupInfoCCInfoReport extends AssociationGroupInfoCC {
 		for (let i = 0; i < this.groups.length; i++) {
 			const offset = 1 + i * 7;
 			this.payload[offset] = this.groups[i].groupId;
-			this.payload.writeUint16BE(this.groups[i].profile, offset + 2);
+			this.payload.writeUInt16BE(this.groups[i].profile, offset + 2);
 			// The remaining bytes are zero
 		}
 
@@ -754,11 +755,11 @@ export class AssociationGroupInfoCCInfoGet extends AssociationGroupInfoCC {
 	public listMode?: boolean;
 	public groupId?: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
+	public serialize(ctx: CCEncodingContext): Bytes {
 		const isListMode = this.listMode === true;
 		const optionByte = (this.refreshCache ? 0b1000_0000 : 0)
 			| (isListMode ? 0b0100_0000 : 0);
-		this.payload = Buffer.from([
+		this.payload = Bytes.from([
 			optionByte,
 			isListMode ? 0 : this.groupId!,
 		]);
@@ -836,10 +837,10 @@ export class AssociationGroupInfoCCCommandListReport
 	)
 	public readonly commands: ReadonlyMap<CommandClasses, readonly number[]>;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
+	public serialize(ctx: CCEncodingContext): Bytes {
 		// To make it easier to encode possible extended CCs, we first
 		// allocate as much space as we may need, then trim it again
-		this.payload = Buffer.allocUnsafe(2 + this.commands.size * 3);
+		this.payload = new Bytes(2 + this.commands.size * 3);
 		this.payload[0] = this.groupId;
 		let offset = 2;
 		for (const [ccId, commands] of this.commands) {
@@ -912,8 +913,8 @@ export class AssociationGroupInfoCCCommandListGet
 	public allowCache: boolean;
 	public groupId: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([
 			this.allowCache ? 0b1000_0000 : 0,
 			this.groupId,
 		]);

packages/cc/src/cc/BarrierOperatorCC.ts

@@ -20,6 +20,7 @@ import type {
 	CCParsingContext,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import {
 	getEnumMemberName,
 	isEnumMember,
@@ -571,8 +572,8 @@ export class BarrierOperatorCCSet extends BarrierOperatorCC {
 
 	public targetState: BarrierState.Open | BarrierState.Closed;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.targetState]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.targetState]);
 		return super.serialize(ctx);
 	}
 
@@ -757,8 +758,8 @@ export class BarrierOperatorCCEventSignalingSet extends BarrierOperatorCC {
 	public subsystemType: SubsystemType;
 	public subsystemState: SubsystemState;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.subsystemType, this.subsystemState]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.subsystemType, this.subsystemState]);
 		return super.serialize(ctx);
 	}
 
@@ -877,8 +878,8 @@ export class BarrierOperatorCCEventSignalingGet extends BarrierOperatorCC {
 
 	public subsystemType: SubsystemType;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.subsystemType]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.subsystemType]);
 		return super.serialize(ctx);
 	}
 

packages/cc/src/cc/BasicCC.ts

@@ -27,6 +27,7 @@ import type {
 	GetSupportedCCVersion,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import { pick } from "@zwave-js/shared/safe";
 import { validateArgs } from "@zwave-js/transformers";
 import {
@@ -396,8 +397,8 @@ export class BasicCCSet extends BasicCC {
 
 	public targetValue: number;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.targetValue]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.targetValue]);
 		return super.serialize(ctx);
 	}
 
@@ -506,8 +507,8 @@ export class BasicCCReport extends BasicCC {
 		return true;
 	}
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([
 			this.currentValue ?? 0xfe,
 			this.targetValue ?? 0xfe,
 			(this.duration ?? Duration.unknown()).serializeReport(),

packages/cc/src/cc/BatteryCC.ts

@@ -26,6 +26,7 @@ import type {
 	GetSupportedCCVersion,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import { type AllOrNone, getEnumMemberName, pick } from "@zwave-js/shared/safe";
 import {
 	CCAPI,
@@ -564,12 +565,12 @@ export class BatteryCCReport extends BatteryCC {
 	@ccValue(BatteryCCValues.lowTemperatureStatus)
 	public readonly lowTemperatureStatus: boolean | undefined;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.isLow ? 0xff : this.level]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.isLow ? 0xff : this.level]);
 		if (this.chargingStatus != undefined) {
-			this.payload = Buffer.concat([
+			this.payload = Bytes.concat([
 				this.payload,
-				Buffer.from([
+				Bytes.from([
 					(this.chargingStatus << 6)
 					+ (this.rechargeable ? 0b0010_0000 : 0)
 					+ (this.backup ? 0b0001_0000 : 0)

packages/cc/src/cc/BinarySensorCC.ts

@@ -16,6 +16,7 @@ import type {
 	CCParsingContext,
 	GetValueDB,
 } from "@zwave-js/host/safe";
+import { Bytes } from "@zwave-js/shared/safe";
 import { getEnumMemberName, isEnumMember } from "@zwave-js/shared/safe";
 import { validateArgs } from "@zwave-js/transformers";
 import {
@@ -399,8 +400,8 @@ export class BinarySensorCCReport extends BinarySensorCC {
 	public type: BinarySensorType;
 	public value: boolean;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.value ? 0xff : 0x00, this.type]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.value ? 0xff : 0x00, this.type]);
 		return super.serialize(ctx);
 	}
 
@@ -459,8 +460,8 @@ export class BinarySensorCCGet extends BinarySensorCC {
 
 	public sensorType: BinarySensorType | undefined;
 
-	public serialize(ctx: CCEncodingContext): Buffer {
-		this.payload = Buffer.from([this.sensorType ?? BinarySensorType.Any]);
+	public serialize(ctx: CCEncodingContext): Bytes {
+		this.payload = Bytes.from([this.sensorType ?? BinarySensorType.Any]);
 		return super.serialize(ctx);
 	}
 
@@ -516,7 +517,7 @@ export class BinarySensorCCSupportedReport extends BinarySensorCC {
 	@ccValue(BinarySensorCCValues.supportedSensorTypes)
 	public supportedSensorTypes: BinarySensorType[];
 
-	public serialize(ctx: CCEncodingContext): Buffer {
+	public serialize(ctx: CCEncodingContext): Bytes {
 		this.payload = encodeBitMask(
 			this.supportedSensorTypes.filter((t) => t !== BinarySensorType.Any),
 			undefined,