From 93f590d1e91c3f78e522b38a11ee6b940b0b5998 Mon Sep 17 00:00:00 2001 From: tsightler Date: Mon, 17 Jul 2023 12:24:22 -0400 Subject: [PATCH 01/14] Retry on failed snapshot requests --- devices/camera.js | 45 ++++++++++++++++++++++++++------------------- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/devices/camera.js b/devices/camera.js index 84d36576..90b14369 100644 --- a/devices/camera.js +++ b/devices/camera.js @@ -719,32 +719,39 @@ export default class Camera extends RingPolledDevice { async refreshSnapshot(type, image_uuid) { let newSnapshot = false + let loop = 3 if (this.device.snapshotsAreBlocked) { this.debug('Snapshots are unavailable, check if motion capture is disabled manually or via modes settings') return } - try { - switch (type) { - case 'interval': - this.debug('Requesting an updated interval snapshot') - newSnapshot = await this.device.getSnapshot() - break; - case 'motion': - if (image_uuid) { - this.debug(`Requesting motion snapshot using notification image UUID: ${image_uuid}`) - newSnapshot = await this.device.getNextSnapshot({ uuid: image_uuid }) - } else if (!this.device.operatingOnBattery) { - this.debug('Requesting an updated motion snapshot') - newSnapshot = await this.device.getNextSnapshot() - } else { - this.debug('Motion snapshot needed but notification did not contain image UUID and battery cameras are unable to snapshot while recording') - } + while (!newSnapshot && loop > 0) { + try { + switch (type) { + case 'interval': + this.debug('Requesting an updated interval snapshot') + newSnapshot = await this.device.getNextSnapshot({ afterMs: Date.now(), maxWaitMs: 5000 }) + break; + case 'motion': + if (image_uuid) { + this.debug(`Requesting motion snapshot using notification image UUID: ${image_uuid}`) + newSnapshot = await this.device.getNextSnapshot({ uuid: image_uuid }) + } else if (!this.device.operatingOnBattery) { + this.debug('Requesting an updated motion snapshot') + newSnapshot = await this.device.getNextSnapshot({ afterMs: Date.now(), maxWaitMs: 5000 }) + } else { + this.debug('Motion snapshot needed but notification did not contain image UUID and battery cameras are unable to snapshot while recording') + } + } + } catch (err) { + this.debug(err) + if (loop > 1) { + this.debug('Failed to retrieve updated snapshot, retrying...') + } else { + this.debug('Failed to retrieve updated snapshot after three attempts, aborting') + } } - } catch (error) { - this.debug(error) - this.debug('Failed to retrieve updated snapshot') } if (newSnapshot) { diff --git a/package-lock.json b/package-lock.json index fa2b7b0f..56fbfbda 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ring-mqtt", - "version": "5.5.0", + "version": "5.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ring-mqtt", - "version": "5.5.0", + "version": "5.5.1", "license": "MIT", "dependencies": { "aedes": "0.49.0", diff --git a/package.json b/package.json index 272ea1ee..74d9ca42 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ring-mqtt", - "version": "5.5.0", + "version": "5.5.1", "type": "module", "description": "Ring Devices via MQTT", "main": "ring-mqtt.js", From 516b29314930f6fbdb518e3c3b5eaa62be486735 Mon Sep 17 00:00:00 2001 From: tsightler Date: Mon, 17 Jul 2023 16:11:01 -0400 Subject: [PATCH 02/14] Always use non-cached snapshot API --- devices/camera.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/devices/camera.js b/devices/camera.js index 90b14369..7f7b9b2e 100644 --- a/devices/camera.js +++ b/devices/camera.js @@ -731,7 +731,7 @@ export default class Camera extends RingPolledDevice { switch (type) { case 'interval': this.debug('Requesting an updated interval snapshot') - newSnapshot = await this.device.getNextSnapshot({ afterMs: Date.now(), maxWaitMs: 5000 }) + newSnapshot = await this.device.getNextSnapshot({ force: true }) break; case 'motion': if (image_uuid) { @@ -739,7 +739,7 @@ export default class Camera extends RingPolledDevice { newSnapshot = await this.device.getNextSnapshot({ uuid: image_uuid }) } else if (!this.device.operatingOnBattery) { this.debug('Requesting an updated motion snapshot') - newSnapshot = await this.device.getNextSnapshot({ afterMs: Date.now(), maxWaitMs: 5000 }) + newSnapshot = await this.device.getNextSnapshot({ force: true }) } else { this.debug('Motion snapshot needed but notification did not contain image UUID and battery cameras are unable to snapshot while recording') } @@ -747,11 +747,13 @@ export default class Camera extends RingPolledDevice { } catch (err) { this.debug(err) if (loop > 1) { - this.debug('Failed to retrieve updated snapshot, retrying...') + this.debug('Failed to retrieve updated snapshot, retrying in one second...') + await utils.sleep(1) } else { this.debug('Failed to retrieve updated snapshot after three attempts, aborting') } } + loop-- } if (newSnapshot) { From 2ca884d33f6b0f7cddc641f0dc9bd8550ac65930 Mon Sep 17 00:00:00 2001 From: tsightler Date: Mon, 17 Jul 2023 16:12:49 -0400 Subject: [PATCH 03/14] Don't retry for motion snapshots with battery cameras --- devices/camera.js | 1 + 1 file changed, 1 insertion(+) diff --git a/devices/camera.js b/devices/camera.js index 7f7b9b2e..7cebfb04 100644 --- a/devices/camera.js +++ b/devices/camera.js @@ -742,6 +742,7 @@ export default class Camera extends RingPolledDevice { newSnapshot = await this.device.getNextSnapshot({ force: true }) } else { this.debug('Motion snapshot needed but notification did not contain image UUID and battery cameras are unable to snapshot while recording') + loop = 0 // Don't retry in this case } } } catch (err) { From 757da3d3f7384bc4add974906555221be01ac019 Mon Sep 17 00:00:00 2001 From: tsightler Date: Mon, 17 Jul 2023 19:50:27 -0400 Subject: [PATCH 04/14] Release v5.5.1 --- docs/CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2bcd0814..60ae1749 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,3 +1,8 @@ +## v5.5.1 +**Bugs Fixed** +- Use non-cached snapshot for all cases. +- Implement multiple retries if initial request for snapshot update fails + ## v5.5.0 **New Features** - Initial support for HEVC mode cameras\ From 252cc74b5b85fe0b6f7321760e283e09dbefe5c6 Mon Sep 17 00:00:00 2001 From: tsightler Date: Wed, 19 Jul 2023 12:10:31 -0400 Subject: [PATCH 05/14] More HEVC Testing --- devices/camera-livestream.js | 12 +----------- devices/camera.js | 27 ++++++++++++++++++++++----- devices/security-panel.js | 3 +-- lib/streaming/peer-connection.js | 2 ++ lib/streaming/ring-edge-connection.js | 14 +++++--------- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/devices/camera-livestream.js b/devices/camera-livestream.js index 2250a24a..b20a814a 100644 --- a/devices/camera-livestream.js +++ b/devices/camera-livestream.js @@ -65,17 +65,7 @@ async function startLiveStream(streamData) { '-c:a:1', 'copy', ], video: [ - ...streamData.hevcEnabled - ? [ - '-c:v', 'libx264', - '-g', '20', - '-keyint_min', '10', - '-crf', '23', - '-preset', 'ultrafast' - ] - : [ - '-c:v', 'copy' - ] + '-c:v', 'copy' ], output: [ '-flags', '+global_header', diff --git a/devices/camera.js b/devices/camera.js index 7cebfb04..9f4d1ee0 100644 --- a/devices/camera.js +++ b/devices/camera.js @@ -767,18 +767,32 @@ export default class Camera extends RingPolledDevice { async startLiveStream(rtspPublishUrl) { this.data.stream.live.session = true + const streamData = { rtspPublishUrl, sessionId: false, - authToken: false, - hevcEnabled: this.hevcEnabled + authToken: false } + try { - if (this.device.isRingEdgeEnabled) { + this.debug('Initializing a live stream WebRTC signaling session') + const response = await this.device.restClient.request({ + method: 'POST', + url: 'https://app.ring.com/api/v1/clap/ticket/request/signalsocket' + }) + streamData.authToken = response.ticket + /* + if (!this.device.isRingEdgeEnabled) { this.debug('Initializing a live stream session for Ring Edge') const auth = await this.device.restClient.getCurrentAuth() streamData.authToken = auth.access_token + const response = await this.device.restClient.request({ + method: 'POST', + url: 'https://app.ring.com/api/v1/clap/ticket/request/signalsocket' + }) + streamData.authToken = response.ticket + } else { this.debug('Initializing a live stream session for Ring cloud') const liveCall = await this.device.restClient.request({ @@ -789,6 +803,7 @@ export default class Camera extends RingPolledDevice { streamData.sessionId = liveCall.data.session_id } } + */ } catch(error) { if (error?.response?.statusCode === 403) { this.debug(`Camera returned 403 when starting a live stream. This usually indicates that live streaming is blocked by Modes settings. Check your Ring app and verify that you are able to stream from this camera with the current Modes settings.`) @@ -798,7 +813,7 @@ export default class Camera extends RingPolledDevice { } if (streamData.sessionId || streamData.authToken) { - this.debug('Live stream session successfully initialized, starting worker') + this.debug('Live stream WebRTC signalling session ticket successfully acquired, starting live stream worker') this.data.stream.live.worker.postMessage({ command: 'start', streamData }) } else { this.debug('Live stream activation failed to initialize session data') @@ -825,7 +840,9 @@ export default class Camera extends RingPolledDevice { try { if (this.data.event_select.transcoded || this.hevcEnabled) { - // Ring videos transcoded for download are poorly optimized for RTSP streaming so they must be re-encoded on-the-fly + // If camera is in HEVC mode, recordings are also in HEVC so transcode the video back to H.264/AVC on the fly + // Ring videos transcoded for download are not optimized for RTSP streaming (limited keyframes) so they must + // also be re-transcoded on-the-fly to allow streamers to join early this.data.stream.event.session = spawn(pathToFfmpeg, [ '-re', '-i', this.data.event_select.recordingUrl, diff --git a/devices/security-panel.js b/devices/security-panel.js index fbd1d696..208a36cf 100644 --- a/devices/security-panel.js +++ b/devices/security-panel.js @@ -177,8 +177,7 @@ export default class SecurityPanel extends RingSocketDevice { this.data.attributes.exitSecondsLeft = 0 } - // Sometimes a countdown event comes in just before the mode switch event - // so suppress publish of attribute state in this case + // Suppress attribute publish if countdown event comes before mode switch if (this.data.publishedState !== this.data.attributes.targetState) { this.publishAlarmAttributes() } diff --git a/lib/streaming/peer-connection.js b/lib/streaming/peer-connection.js index 221d5b17..df8575d1 100644 --- a/lib/streaming/peer-connection.js +++ b/lib/streaming/peer-connection.js @@ -53,6 +53,7 @@ export class WeriftPeerConnection extends Subscribed { ], parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1', }), + /* new RTCRtpCodecParameters({ mimeType: 'video/H265', clockRate: 90000, @@ -64,6 +65,7 @@ export class WeriftPeerConnection extends Subscribed { { type: 'goog-remb' }, ], }), + */ new RTCRtpCodecParameters({ mimeType: "video/rtx", clockRate: 90000, diff --git a/lib/streaming/ring-edge-connection.js b/lib/streaming/ring-edge-connection.js index 8ba7f8a2..da664301 100644 --- a/lib/streaming/ring-edge-connection.js +++ b/lib/streaming/ring-edge-connection.js @@ -11,19 +11,15 @@ import crypto from 'crypto' export class RingEdgeConnection extends StreamingConnectionBase { constructor(authToken, camera) { super( - new WebSocket('wss://api.prod.signalling.ring.devices.a2z.com:443/ws', { + new WebSocket(`wss://api.prod.signalling.ring.devices.a2z.com:443/ws?api_version=4.0&auth_type=ring_solutions&client_id=ring_site-${crypto.randomUUID()}&token=${authToken}`, { headers: { - Authorization: `Bearer ${authToken}`, - 'X-Sig-API-Version': '4.0', - 'X-Sig-Client-ID': `ring_android-${crypto - .randomBytes(4) - .toString('hex')}`, - 'X-Sig-Client-Info': 'Ring/3.60.0;Platform/Android;OS/12;Density/2.75;Device/samsung-SM-T710;Locale/en-US;TimeZone/GMT-07:00', - 'X-Sig-Auth-Type': 'ring_oauth', + // This must exist but the contents do not seem to matter, however, decided to use the + // Firefox default since it doesn't also doesn't support H.265/HEVC + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0' } }) - ) + ) this.camera = camera this.onSessionId = new ReplaySubject(1) this.onOfferSent = new ReplaySubject(1) From 6dd6e5ad8d1dd4d376b03c489ddf0ae6179e33ba Mon Sep 17 00:00:00 2001 From: tsightler Date: Wed, 19 Jul 2023 14:32:49 -0400 Subject: [PATCH 06/14] Streaming changes for HEVC cameras --- devices/camera-livestream.js | 5 +- devices/camera.js | 36 +---- lib/streaming/peer-connection.js | 15 +- lib/streaming/ring-edge-connection.js | 150 -------------------- lib/streaming/streaming-connection-base.js | 2 +- lib/streaming/streaming-session.js | 2 +- lib/streaming/subscribed.js | 2 +- lib/streaming/webrtc-connection.js | 156 ++++++++++++++++----- 8 files changed, 131 insertions(+), 237 deletions(-) delete mode 100644 lib/streaming/ring-edge-connection.js diff --git a/devices/camera-livestream.js b/devices/camera-livestream.js index b20a814a..6932792c 100644 --- a/devices/camera-livestream.js +++ b/devices/camera-livestream.js @@ -1,6 +1,5 @@ import { parentPort, workerData } from 'worker_threads' import { WebrtcConnection } from '../lib/streaming/webrtc-connection.js' -import { RingEdgeConnection } from '../lib/streaming/ring-edge-connection.js' import { StreamingSession } from '../lib/streaming/streaming-session.js' const deviceName = workerData.deviceName @@ -31,9 +30,7 @@ async function startLiveStream(streamData) { id: doorbotId } - const streamConnection = (streamData.sessionId) - ? new WebrtcConnection(streamData.sessionId, cameraData) - : new RingEdgeConnection(streamData.authToken, cameraData) + const streamConnection = new WebrtcConnection(streamData.ticket, cameraData) liveStream = new StreamingSession(cameraData, streamConnection) liveStream.connection.pc.onConnectionState.subscribe(async (data) => { diff --git a/devices/camera.js b/devices/camera.js index 9f4d1ee0..ce14a53a 100644 --- a/devices/camera.js +++ b/devices/camera.js @@ -770,40 +770,16 @@ export default class Camera extends RingPolledDevice { const streamData = { rtspPublishUrl, - sessionId: false, - authToken: false + ticket: null } - try { - this.debug('Initializing a live stream WebRTC signaling session') + this.debug('Acquiring a live stream WebRTC signaling session ticket') const response = await this.device.restClient.request({ method: 'POST', url: 'https://app.ring.com/api/v1/clap/ticket/request/signalsocket' }) - streamData.authToken = response.ticket - /* - if (!this.device.isRingEdgeEnabled) { - this.debug('Initializing a live stream session for Ring Edge') - const auth = await this.device.restClient.getCurrentAuth() - streamData.authToken = auth.access_token - const response = await this.device.restClient.request({ - method: 'POST', - url: 'https://app.ring.com/api/v1/clap/ticket/request/signalsocket' - }) - streamData.authToken = response.ticket - - } else { - this.debug('Initializing a live stream session for Ring cloud') - const liveCall = await this.device.restClient.request({ - method: 'POST', - url: this.device.doorbotUrl('live_call') - }) - if (liveCall.data?.session_id) { - streamData.sessionId = liveCall.data.session_id - } - } - */ + streamData.ticket = response.ticket } catch(error) { if (error?.response?.statusCode === 403) { this.debug(`Camera returned 403 when starting a live stream. This usually indicates that live streaming is blocked by Modes settings. Check your Ring app and verify that you are able to stream from this camera with the current Modes settings.`) @@ -812,11 +788,11 @@ export default class Camera extends RingPolledDevice { } } - if (streamData.sessionId || streamData.authToken) { - this.debug('Live stream WebRTC signalling session ticket successfully acquired, starting live stream worker') + if (streamData.ticket) { + this.debug('Live stream WebRTC signaling session ticket acquired, starting live stream worker') this.data.stream.live.worker.postMessage({ command: 'start', streamData }) } else { - this.debug('Live stream activation failed to initialize session data') + this.debug('Live stream failed to initialize WebRTC signaling session') this.data.stream.live.status = 'failed' this.data.stream.live.session = false this.publishStreamState() diff --git a/lib/streaming/peer-connection.js b/lib/streaming/peer-connection.js index df8575d1..cf7713c2 100644 --- a/lib/streaming/peer-connection.js +++ b/lib/streaming/peer-connection.js @@ -1,5 +1,5 @@ // This code is largely copied from ring-client-api, but converted from Typescript -// to straight Javascript and some code not required for ring-mqtt removed. +// to native Javascript with custom logging for ring-mqtt and some unused code removed. // Much thanks to @dgreif for the original code which is the basis for this work. import { RTCPeerConnection, RTCRtpCodecParameters } from 'werift' @@ -53,19 +53,6 @@ export class WeriftPeerConnection extends Subscribed { ], parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1', }), - /* - new RTCRtpCodecParameters({ - mimeType: 'video/H265', - clockRate: 90000, - rtcpFeedback: [ - { type: 'transport-cc' }, - { type: 'ccm', parameter: 'fir' }, - { type: 'nack' }, - { type: 'nack', parameter: 'pli' }, - { type: 'goog-remb' }, - ], - }), - */ new RTCRtpCodecParameters({ mimeType: "video/rtx", clockRate: 90000, diff --git a/lib/streaming/ring-edge-connection.js b/lib/streaming/ring-edge-connection.js deleted file mode 100644 index da664301..00000000 --- a/lib/streaming/ring-edge-connection.js +++ /dev/null @@ -1,150 +0,0 @@ -// This code is largely copied from ring-client-api, but converted from Typescript -// to straight Javascript and some code not required for ring-mqtt removed. -// Much thanks to @dgreif for the original code which is the basis for this work. - -import WebSocket from 'ws' -import { parentPort } from 'worker_threads' -import { firstValueFrom, interval, ReplaySubject } from 'rxjs' -import { StreamingConnectionBase } from './streaming-connection-base.js' -import crypto from 'crypto' - -export class RingEdgeConnection extends StreamingConnectionBase { - constructor(authToken, camera) { - super( - new WebSocket(`wss://api.prod.signalling.ring.devices.a2z.com:443/ws?api_version=4.0&auth_type=ring_solutions&client_id=ring_site-${crypto.randomUUID()}&token=${authToken}`, { - headers: { - // This must exist but the contents do not seem to matter, however, decided to use the - // Firefox default since it doesn't also doesn't support H.265/HEVC - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0' - } - }) - - ) - this.camera = camera - this.onSessionId = new ReplaySubject(1) - this.onOfferSent = new ReplaySubject(1) - this.sessionId = null - this.dialogId = `${crypto.randomUUID()}-${Math.floor(100000 + Math.random() * 900000)}` - - this.addSubscriptions( - this.onWsOpen.subscribe(() => { - parentPort.postMessage({type: 'log_info', data: 'Websocket signalling for Ring Edge connected successfully'}) - this.initiateCall().catch((error) => { - parentPort.postMessage({type: 'log_error', data: error}) - this.callEnded() - }) - }), - - // The ring-edge session needs a ping every 5 seconds to keep the connection alive - interval(5000).subscribe(() => { - this.sendSessionMessage('ping') - }), - - this.pc.onIceCandidate.subscribe(async (iceCandidate) => { - await firstValueFrom(this.onOfferSent) - this.sendMessage({ - body: { - doorbot_id: camera.id, - ice: iceCandidate.candidate, - mlineindex: iceCandidate.sdpMLineIndex, - }, - dialog_id: this.dialogId, - method: 'ice', - }) - }) - ) - } - - async initiateCall() { - const { sdp } = await this.pc.createOffer() - - this.sendMessage({ - body: { - doorbot_id: this.camera.id, - stream_options: { audio_enabled: true, video_enabled: true }, - sdp, - }, - dialog_id: this.dialogId, - method: 'live_view' - }) - - this.onOfferSent.next() - } - - async handleMessage(message) { - if (message.body.doorbot_id !== this.camera.id) { - // ignore messages for other cameras - return - } - - if (['session_created', 'session_started'].includes(message.method) && - 'session_id' in message.body && - !this.sessionId - ) { - this.sessionId = message.body.session_id - this.onSessionId.next(this.sessionId) - } - - if (message.body.session_id && message.body.session_id !== this.sessionId) { - // ignore messages for other sessions - return - } - - switch (message.method) { - case 'session_created': - case 'session_started': - // session already stored above - return - case 'sdp': - await this.pc.acceptAnswer(message.body) - this.onCallAnswered.next(message.body.sdp) - this.activate() - return - case 'ice': - await this.pc.addIceCandidate({ - candidate: message.body.ice, - sdpMLineIndex: message.body.mlineindex, - }) - return - case 'pong': - return - case 'notification': - const { text } = message.body - if (text === 'PeerConnectionState::kConnecting' || - text === 'PeerConnectionState::kConnected') { - return - } - break - case 'close': - this.callEnded() - return - } - } - - sendSessionMessage(method, body = {}) { - const sendSessionMessage = () => { - const message = { - body: { - ...body, - doorbot_id: this.camera.id, - session_id: this.sessionId, - }, - dialog_id: this.dialogId, - method - } - this.sendMessage(message) - } - if (this.sessionId) { - // Send immediately if we already have a session id - // This is needed to send `close` before closing the websocket - sendSessionMessage() - } - else { - firstValueFrom(this.onSessionId) - .then(sendSessionMessage) - .catch((e) => { - // debug(e) - }) - } - } -} diff --git a/lib/streaming/streaming-connection-base.js b/lib/streaming/streaming-connection-base.js index f33257b8..f5ec3e4a 100644 --- a/lib/streaming/streaming-connection-base.js +++ b/lib/streaming/streaming-connection-base.js @@ -1,5 +1,5 @@ // This code is largely copied from ring-client-api, but converted from Typescript -// to straight Javascript and some code not required for ring-mqtt removed. +// to native Javascript with custom logging for ring-mqtt and some unused code removed. // Much thanks to @dgreif for the original code which is the basis for this work. import { WeriftPeerConnection } from './peer-connection.js' diff --git a/lib/streaming/streaming-session.js b/lib/streaming/streaming-session.js index 1f0eeea7..843d8474 100644 --- a/lib/streaming/streaming-session.js +++ b/lib/streaming/streaming-session.js @@ -1,5 +1,5 @@ // This code is largely copied from ring-client-api, but converted from Typescript -// to straight Javascript and some code not required for ring-mqtt removed. +// to native Javascript with custom logging for ring-mqtt and some unused code removed. // Much thanks to @dgreif for the original code which is the basis for this work. import { FfmpegProcess, reservePorts, RtpSplitter, } from '@homebridge/camera-utils' diff --git a/lib/streaming/subscribed.js b/lib/streaming/subscribed.js index 107b8b40..7fe36c90 100644 --- a/lib/streaming/subscribed.js +++ b/lib/streaming/subscribed.js @@ -1,5 +1,5 @@ // This code is largely copied from ring-client-api, but converted from Typescript -// to straight Javascript and some code not required for ring-mqtt removed. +// to native Javascript with custom logging for ring-mqtt and some unused code removed. // Much thanks to @dgreif for the original code which is the basis for this work. export class Subscribed { diff --git a/lib/streaming/webrtc-connection.js b/lib/streaming/webrtc-connection.js index cbf73b40..ba179bca 100644 --- a/lib/streaming/webrtc-connection.js +++ b/lib/streaming/webrtc-connection.js @@ -1,66 +1,150 @@ // This code is largely copied from ring-client-api, but converted from Typescript -// to straight Javascript and some code not required for ring-mqtt removed. +// to native Javascript with custom logging for ring-mqtt and some unused code removed. // Much thanks to @dgreif for the original code which is the basis for this work. import WebSocket from 'ws' import { parentPort } from 'worker_threads' +import { firstValueFrom, interval, ReplaySubject } from 'rxjs' import { StreamingConnectionBase } from './streaming-connection-base.js' +import crypto from 'crypto' -function parseLiveCallSession(sessionId) { - const encodedSession = sessionId.split('.')[1] - const buff = Buffer.from(encodedSession, 'base64') - const text = buff.toString('ascii') - return JSON.parse(text) -} - -export class WebrtcConnection extends StreamingConnectionBase { - constructor(sessionId, camera) { - const liveCallSession = parseLiveCallSession(sessionId) - - super(new WebSocket(`wss://${liveCallSession.rms_fqdn}:${liveCallSession.webrtc_port}/`, { - headers: { - API_VERSION: '3.1', - API_TOKEN: sessionId, - CLIENT_INFO: 'Ring/3.49.0;Platform/Android;OS/7.0;Density/2.0;Device/samsung-SM-T710;Locale/en-US;TimeZone/GMT-07:00', - }, - })) +export class RingEdgeConnection extends StreamingConnectionBase { + constructor(ticket, camera) { + super( + new WebSocket(`wss://api.prod.signalling.ring.devices.a2z.com:443/ws?api_version=4.0&auth_type=ring_solutions&client_id=ring_site-${crypto.randomUUID()}&token=${ticket}`, { + headers: { + // This must exist but the contents do not seem to matter, however, decided to use the + // Firefox default since it doesn't also doesn't support H.265/HEVC + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0' + } + }) + ) this.camera = camera - this.sessionId = sessionId + this.onSessionId = new ReplaySubject(1) + this.onOfferSent = new ReplaySubject(1) + this.sessionId = null + this.dialogId = `${crypto.randomUUID()}-${Math.floor(100000 + Math.random() * 900000)}` this.addSubscriptions( this.onWsOpen.subscribe(() => { - parentPort.postMessage({type: 'log_info', data: 'Websocket signaling for Ring cloud connected successfully'}) + parentPort.postMessage({type: 'log_info', data: 'Websocket signalling for Ring Edge connected successfully'}) + this.initiateCall().catch((error) => { + parentPort.postMessage({type: 'log_error', data: error}) + this.callEnded() + }) + }), + + // The ring-edge session needs a ping every 5 seconds to keep the connection alive + interval(5000).subscribe(() => { + this.sendSessionMessage('ping') + }), + + this.pc.onIceCandidate.subscribe(async (iceCandidate) => { + await firstValueFrom(this.onOfferSent) + this.sendMessage({ + body: { + doorbot_id: camera.id, + ice: iceCandidate.candidate, + mlineindex: iceCandidate.sdpMLineIndex, + }, + dialog_id: this.dialogId, + method: 'ice', + }) }) ) } + async initiateCall() { + const { sdp } = await this.pc.createOffer() + + this.sendMessage({ + body: { + doorbot_id: this.camera.id, + stream_options: { audio_enabled: true, video_enabled: true }, + sdp, + }, + dialog_id: this.dialogId, + method: 'live_view' + }) + + this.onOfferSent.next() + } + async handleMessage(message) { + if (message.body.doorbot_id !== this.camera.id) { + // ignore messages for other cameras + return + } + + if (['session_created', 'session_started'].includes(message.method) && + 'session_id' in message.body && + !this.sessionId + ) { + this.sessionId = message.body.session_id + this.onSessionId.next(this.sessionId) + } + + if (message.body.session_id && message.body.session_id !== this.sessionId) { + // ignore messages for other sessions + return + } + switch (message.method) { + case 'session_created': + case 'session_started': + // session already stored above + return case 'sdp': - try { - const answer = await this.pc.createAnswer(message) - this.sendSessionMessage('sdp', answer) - this.onCallAnswered.next(message.sdp) - this.activate() - } catch(err) { - parentPort.postMessage({type: 'log_error', data: 'ERROR - Unable to negotiate H.264 codec, verify this camera has Legacy Video Mode enabled'}) - this.pc.close() - } + await this.pc.acceptAnswer(message.body) + this.onCallAnswered.next(message.body.sdp) + this.activate() return case 'ice': await this.pc.addIceCandidate({ - candidate: message.ice, - sdpMLineIndex: message.mlineindex, + candidate: message.body.ice, + sdpMLineIndex: message.body.mlineindex, }) return + case 'pong': + return + case 'notification': + const { text } = message.body + if (text === 'PeerConnectionState::kConnecting' || + text === 'PeerConnectionState::kConnected') { + return + } + break + case 'close': + this.callEnded() + return } } sendSessionMessage(method, body = {}) { - this.sendMessage({ - ...body, - method, - }) + const sendSessionMessage = () => { + const message = { + body: { + ...body, + doorbot_id: this.camera.id, + session_id: this.sessionId, + }, + dialog_id: this.dialogId, + method + } + this.sendMessage(message) + } + if (this.sessionId) { + // Send immediately if we already have a session id + // This is needed to send `close` before closing the websocket + sendSessionMessage() + } + else { + firstValueFrom(this.onSessionId) + .then(sendSessionMessage) + .catch((e) => { + // debug(e) + }) + } } } From f0be34a260b83e89ca902b45680ccc51a8ff10c8 Mon Sep 17 00:00:00 2001 From: tsightler Date: Wed, 19 Jul 2023 14:34:09 -0400 Subject: [PATCH 07/14] More HEVC changes --- lib/streaming/webrtc-connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/streaming/webrtc-connection.js b/lib/streaming/webrtc-connection.js index ba179bca..64e647e0 100644 --- a/lib/streaming/webrtc-connection.js +++ b/lib/streaming/webrtc-connection.js @@ -8,7 +8,7 @@ import { firstValueFrom, interval, ReplaySubject } from 'rxjs' import { StreamingConnectionBase } from './streaming-connection-base.js' import crypto from 'crypto' -export class RingEdgeConnection extends StreamingConnectionBase { +export class WebrtcConnection extends StreamingConnectionBase { constructor(ticket, camera) { super( new WebSocket(`wss://api.prod.signalling.ring.devices.a2z.com:443/ws?api_version=4.0&auth_type=ring_solutions&client_id=ring_site-${crypto.randomUUID()}&token=${ticket}`, { From 73038a20cb49311deb215748deb2155b00baf28d Mon Sep 17 00:00:00 2001 From: tsightler Date: Wed, 19 Jul 2023 14:55:57 -0400 Subject: [PATCH 08/14] Release v5.5.1 --- docs/CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 60ae1749..4cf1e62b 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,9 @@ ## v5.5.1 -**Bugs Fixed** +**New Features** +- Improved support for HEVC mode cameras\ + While the initial support for HEVC required transcoding, this new version uses a different Ring streaming API that negotiates down to H.264/AVC on-the-fly which means these camera should now work fine even on lower-performance hardware like RPi3/4 devices. Hopefully this new API does not break streaming for other cases. + +**Other Changes** - Use non-cached snapshot for all cases. - Implement multiple retries if initial request for snapshot update fails From 7fd11cc7cd3f184de554c5ca4021190a313c44dd Mon Sep 17 00:00:00 2001 From: tsightler Date: Wed, 19 Jul 2023 21:48:02 -0400 Subject: [PATCH 09/14] Reduce H.264 profile to baseline level 3.1 --- lib/streaming/peer-connection.js | 8 ++------ lib/streaming/webrtc-connection.js | 5 +++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/streaming/peer-connection.js b/lib/streaming/peer-connection.js index cf7713c2..ac802b89 100644 --- a/lib/streaming/peer-connection.js +++ b/lib/streaming/peer-connection.js @@ -51,16 +51,12 @@ export class WeriftPeerConnection extends Subscribed { { type: 'nack', parameter: 'pli' }, { type: 'goog-remb' }, ], - parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1', + parameters: 'packetization-mode=1;profile-level-id=42001f;level-asymmetry-allowed=1', }), new RTCRtpCodecParameters({ mimeType: "video/rtx", clockRate: 90000, - }), - new RTCRtpCodecParameters({ - mimeType: "video/red", - clockRate: 90000, - }), + }) ], }, iceServers: ringIceServers.map((server) => ({ urls: server })), diff --git a/lib/streaming/webrtc-connection.js b/lib/streaming/webrtc-connection.js index 64e647e0..d34c9c82 100644 --- a/lib/streaming/webrtc-connection.js +++ b/lib/streaming/webrtc-connection.js @@ -13,8 +13,9 @@ export class WebrtcConnection extends StreamingConnectionBase { super( new WebSocket(`wss://api.prod.signalling.ring.devices.a2z.com:443/ws?api_version=4.0&auth_type=ring_solutions&client_id=ring_site-${crypto.randomUUID()}&token=${ticket}`, { headers: { - // This must exist but the contents do not seem to matter, however, decided to use the - // Firefox default since it doesn't also doesn't support H.265/HEVC + // This must exist or the socket will close immediately but the contents do not seem + // to matter, however, I decided to use the Firefox default user agent since Firefox + // doesn't explicitly support H.265/HEVC 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0' } }) From df64b33ddfa2d5c0ce744f9bc73431d755460470 Mon Sep 17 00:00:00 2001 From: tsightler Date: Thu, 20 Jul 2023 11:13:43 -0400 Subject: [PATCH 10/14] Release v5.5.1 --- Dockerfile | 2 +- docs/CHANGELOG.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e4d6f844..4dd7e4e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ENV LANG="C.UTF-8" \ COPY . /app/ring-mqtt RUN S6_VERSION="v3.1.5.0" && \ BASHIO_VERSION="v0.15.0" && \ - GO2RTC_VERSION="v1.6.0" && \ + GO2RTC_VERSION="v1.6.1" && \ APK_ARCH="$(apk --print-arch)" && \ apk add --no-cache tar xz git libcrypto3 libssl3 musl-utils musl bash curl jq tzdata nodejs npm mosquitto-clients && \ curl -L -s "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-noarch.tar.xz" | tar -Jxpf - -C / && \ diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4cf1e62b..888951c5 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,9 @@ - Use non-cached snapshot for all cases. - Implement multiple retries if initial request for snapshot update fails +**Dependency Updates** +- go2rtc v1.6.1 + ## v5.5.0 **New Features** - Initial support for HEVC mode cameras\ From 22fa49eb29e5389d8356ac47e3bee93df8cba23c Mon Sep 17 00:00:00 2001 From: tsightler Date: Thu, 20 Jul 2023 13:32:30 -0400 Subject: [PATCH 11/14] Fix spelling inconsistency --- lib/streaming/webrtc-connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/streaming/webrtc-connection.js b/lib/streaming/webrtc-connection.js index d34c9c82..14b60730 100644 --- a/lib/streaming/webrtc-connection.js +++ b/lib/streaming/webrtc-connection.js @@ -29,7 +29,7 @@ export class WebrtcConnection extends StreamingConnectionBase { this.addSubscriptions( this.onWsOpen.subscribe(() => { - parentPort.postMessage({type: 'log_info', data: 'Websocket signalling for Ring Edge connected successfully'}) + parentPort.postMessage({type: 'log_info', data: 'Websocket signaling for Ring Edge connected successfully'}) this.initiateCall().catch((error) => { parentPort.postMessage({type: 'log_error', data: error}) this.callEnded() From a50726db27e4a20213881a561cf0b28649bb3e26 Mon Sep 17 00:00:00 2001 From: tsightler Date: Thu, 20 Jul 2023 20:33:33 -0400 Subject: [PATCH 12/14] Release v5.5.1 --- Dockerfile | 2 +- docs/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4dd7e4e8..120ee942 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ ENV LANG="C.UTF-8" \ COPY . /app/ring-mqtt RUN S6_VERSION="v3.1.5.0" && \ BASHIO_VERSION="v0.15.0" && \ - GO2RTC_VERSION="v1.6.1" && \ + GO2RTC_VERSION="v1.6.2" && \ APK_ARCH="$(apk --print-arch)" && \ apk add --no-cache tar xz git libcrypto3 libssl3 musl-utils musl bash curl jq tzdata nodejs npm mosquitto-clients && \ curl -L -s "https://github.com/just-containers/s6-overlay/releases/download/${S6_VERSION}/s6-overlay-noarch.tar.xz" | tar -Jxpf - -C / && \ diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 888951c5..10da5688 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,7 +1,7 @@ ## v5.5.1 **New Features** - Improved support for HEVC mode cameras\ - While the initial support for HEVC required transcoding, this new version uses a different Ring streaming API that negotiates down to H.264/AVC on-the-fly which means these camera should now work fine even on lower-performance hardware like RPi3/4 devices. Hopefully this new API does not break streaming for other cases. + While the initial support for HEVC required local transcoding, this update uses a different streaming API that is able to negotiates down to H.264/AVC for these cameras on-the-fly which means these camera should now work fine even on lower-performance hardware like RPi3/4 devices. Hopefully this new API does not break streaming for other cases. **Other Changes** - Use non-cached snapshot for all cases. From c8d36443dd52d55129bd5f2065a784e3d55b2af9 Mon Sep 17 00:00:00 2001 From: tsightler Date: Tue, 25 Jul 2023 12:20:37 -0400 Subject: [PATCH 13/14] Release v5.5.1 --- docs/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 10da5688..45616c26 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -8,7 +8,7 @@ - Implement multiple retries if initial request for snapshot update fails **Dependency Updates** -- go2rtc v1.6.1 +- go2rtc v1.6.2 ## v5.5.0 **New Features** From 643a15dc88aeec41362cccacf4a76dd59ad12cbf Mon Sep 17 00:00:00 2001 From: tsightler Date: Tue, 25 Jul 2023 12:21:43 -0400 Subject: [PATCH 14/14] Release v5.5.1 --- docs/CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 45616c26..f3342551 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,10 +1,10 @@ ## v5.5.1 **New Features** - Improved support for HEVC mode cameras\ - While the initial support for HEVC required local transcoding, this update uses a different streaming API that is able to negotiates down to H.264/AVC for these cameras on-the-fly which means these camera should now work fine even on lower-performance hardware like RPi3/4 devices. Hopefully this new API does not break streaming for other cases. + While the initial support for HEVC required local transcoding, this update uses a different streaming API that is able to negotiates down to H.264/AVC for these cameras on-the-fly which means HEVC enabled cameras should now work fine even on lower-performance hardware like RPi3/4 devices. Hopefully this new API does not break streaming for other cases. **Other Changes** -- Use non-cached snapshot for all cases. +- Use a non-cached snapshot for all cases - Implement multiple retries if initial request for snapshot update fails **Dependency Updates**