From 79712d870edea74c4503b5eea244811e65a7659d Mon Sep 17 00:00:00 2001 From: Aron Homberg Date: Tue, 5 Nov 2024 23:05:09 +0100 Subject: [PATCH] fix: further stabilizations --- audio-processor.js | 2 +- manifest.json | 4 +- src/data/price-models/anthropic.json | 4 +- src/data/price-models/openai.json | 18 ++++++ src/lib/content-script/llm.ts | 89 ++++++++++++++++++---------- src/lib/worker/message-channel.ts | 64 ++++++++++++++------ src/worker.ts | 79 ++++++++++++++---------- 7 files changed, 176 insertions(+), 84 deletions(-) diff --git a/audio-processor.js b/audio-processor.js index b51bd13..99b2b5f 100644 --- a/audio-processor.js +++ b/audio-processor.js @@ -77,7 +77,7 @@ async function decodeFilterSlice(file) { const gainNode = audioContext.createGain(); oscillator.type = 'sine'; // Use a sine wave - oscillator.frequency.setValueAtTime(24000, audioContext.currentTime); // Set frequency to 24kHz + oscillator.frequency.setValueAtTime(22050, audioContext.currentTime); // Set frequency to 24kHz gainNode.gain.setValueAtTime(0.5, audioContext.currentTime); // Set the volume to very low // Connect oscillator to the gain node and then to the audio output diff --git a/manifest.json b/manifest.json index 01bfd36..d478b7e 100644 --- a/manifest.json +++ b/manifest.json @@ -37,7 +37,7 @@ } ], "host_permissions": ["*://*/*", "http://127.0.0.1/*", "http://localhost/*"], - "permissions": ["webRequest", "scripting", "activeTab", "storage", "unlimitedStorage", "offscreen"], + "permissions": ["webRequest", "scripting", "activeTab", "storage", "unlimitedStorage", "offscreen", "alarms"], "background": { "service_worker": "src/worker.ts", "type": "module" @@ -50,7 +50,7 @@ { "resources": ["message-channel.html", "message-channel.js", "audio-processor.html", "audio-processor.js", "data/OK.mp3", "data/audio_file_processing_de.mp3", "data/audio_file_processing_en.mp3"], "matches": [""], - "use_dynamic_url": true + "use_dynamic_url": false }, { "resources": ["dist/*"], diff --git a/src/data/price-models/anthropic.json b/src/data/price-models/anthropic.json index 7776230..e6654c6 100644 --- a/src/data/price-models/anthropic.json +++ b/src/data/price-models/anthropic.json @@ -1,5 +1,5 @@ { - "anthropic-claude-3-5-sonnet-20240620": { + "anthropic-claude-3-5-sonnet-20241022": { "provider": "anthropic", "input": 0.000003, "output": 0.000015, @@ -17,7 +17,7 @@ "maxOutputTokens": 4096, "tikTokenModelName": "claude-3" }, - "anthropic-claude-3-haiku-20240307": { + "anthropic-claude-3-5-haiku-20241022": { "provider": "anthropic", "input": 0.00000025, "output": 0.00000125, diff --git a/src/data/price-models/openai.json b/src/data/price-models/openai.json index 90ee10c..18af3af 100644 --- a/src/data/price-models/openai.json +++ b/src/data/price-models/openai.json @@ -1,4 +1,22 @@ { + "openai-o1-mini": { + "provider": "openai", + "input": 0.000005, + "output": 0.000015, + "maxContextTokens": 128000, + "maxInputTokens": 123904, + "maxOutputTokens": 4096, + "tikTokenModelName": "o1-mini" + }, + "openai-o1-preview": { + "provider": "openai", + "input": 0.000005, + "output": 0.000015, + "maxContextTokens": 128000, + "maxInputTokens": 123904, + "maxOutputTokens": 4096, + "tikTokenModelName": "o1-preview" + }, "openai-gpt-4o": { "provider": "openai", "input": 0.000005, diff --git a/src/lib/content-script/llm.ts b/src/lib/content-script/llm.ts index e690747..79268e8 100644 --- a/src/lib/content-script/llm.ts +++ b/src/lib/content-script/llm.ts @@ -1,63 +1,85 @@ import { useState, useEffect, useCallback } from "react"; import type { Prompt } from "./prompt-template"; -export function useLlmStreaming({ name, onPayloadReceived }: { name: string, onPayloadReceived: (payload: any) => void }): -(prompt: Prompt) => void { - +/** + * React hook for handling streaming LLM prompts over a persistent connection with the background script. + * @param name A unique name for the connection, identifying the port. + * @param onPayloadReceived Callback function that processes incoming payloads. + * @returns A function to send prompts to the background script. + */ +export function useLlmStreaming({ + name, + onPayloadReceived, +}: { + name: string; + onPayloadReceived: (payload: any) => void; +}): (prompt: Prompt) => void { const [llmPort, setLlmPort] = useState(null); - // ensure llmPort is always connected and disconnects on component rerender + // Ensures llmPort is always connected and reconnects on component re-render or disconnect useEffect(() => { - console.log("useLlmStreaming", name); - setLlmPort((llmPort) => { - if (llmPort === null) { - llmPort = chrome.runtime.connect({ name: `${name}-llm-stream` }); - } - return llmPort; - }); + const connectPort = () => { + console.log("Establishing new port connection:", name); + const port = chrome.runtime.connect({ name: `${name}-llm-stream` }); + + // Detect when the port is disconnected and attempt to reconnect + port.onDisconnect.addListener(() => { + console.log("Port disconnected, attempting to reconnect..."); + setLlmPort(null); // Reset the port state to trigger reconnection + connectPort(); // Reconnect if disconnected + }); + + setLlmPort(port); // Set the new port + }; + + connectPort(); // Establish the initial connection return () => { - setLlmPort((llmPort) => { - if (llmPort) { - llmPort.disconnect(); + // Clean up the port on component unmount + setLlmPort((port) => { + if (port) { + port.disconnect(); + console.log("Port manually disconnected"); } return null; }); }; }, [name]); - const [, setListenerRegistered] = useState(null); + // State to store the currently registered listener function for cleanup + const [, setListenerRegistered] = useState(null); useEffect(() => { - if (llmPort) { setListenerRegistered((listener) => { - + // Remove any existing listener before adding a new one if (typeof listener === "function") { - console.log("removing previous onPortMessageReceived listener", listener); + console.log("Removing previous onPortMessageReceived listener", listener); llmPort.onMessage.removeListener(listener as any); } + + // Define the new listener for handling incoming messages const newListener = (message: { action: string; payload: any }) => { switch (message.action) { case "prompt-response": { - onPayloadReceived(message.payload); + onPayloadReceived(message.payload); // Pass the payload to the callback break; } } }; - console.log("adding new onPortMessageReceived listener", newListener); - llmPort.onMessage.addListener(newListener); + console.log("Adding new onPortMessageReceived listener", newListener); + llmPort.onMessage.addListener(newListener); // Register the new listener return newListener; }); } return () => { + // Clean up listener on component re-render or unmount if (llmPort) { setListenerRegistered((listenerRegistered) => { - if (typeof listenerRegistered === "function") { - console.log("removing onPortMessageReceived listener on destruct", listenerRegistered); + console.log("Removing onPortMessageReceived listener on cleanup", listenerRegistered); llmPort.onMessage.removeListener(listenerRegistered as any); } return null; @@ -66,13 +88,20 @@ export function useLlmStreaming({ name, onPayloadReceived }: { name: string, onP }; }, [llmPort, onPayloadReceived]); - const onPrompt = useCallback((prompt: Prompt) => { - if (llmPort) { - llmPort.postMessage({ action: "prompt", payload: prompt }); - } else { - console.error("llmPort is not connected"); - } - }, [llmPort]); + /** + * Function to send a prompt message through the persistent connection. + * @param prompt The prompt to send to the background script. + */ + const onPrompt = useCallback( + (prompt: Prompt) => { + if (llmPort) { + llmPort.postMessage({ action: "prompt", payload: prompt }); + } else { + console.error("llmPort is not connected"); + } + }, + [llmPort] + ); return onPrompt; } diff --git a/src/lib/worker/message-channel.ts b/src/lib/worker/message-channel.ts index 8196b97..8289a9c 100644 --- a/src/lib/worker/message-channel.ts +++ b/src/lib/worker/message-channel.ts @@ -1,50 +1,76 @@ -declare let self: ServiceWorkerGlobalScope & { tunnelPort: MessagePort }; +// Declare self as a ServiceWorkerGlobalScope with an additional tunnelPort for handling messages +declare let self: ServiceWorkerGlobalScope & { tunnelPort: MessagePort | null }; +// Dictionary to store message listeners by unique keys const tunnelListeners: Record void> = {}; +/** + * Function to set up or retrieve a persistent message channel with a content script. + * @param onMessageHandler Optional function to handle incoming messages to the service worker. + * @returns An object with methods to post messages and manage listeners. + */ export const useMessageChannel = ( onMessageHandler?: typeof self.onmessage, ) => { if (!self.tunnelPort) { - console.log("Making new messagechannel (message-channel.ts, worker)"); + console.log("Creating new message channel (message-channel.ts, worker)"); + + // Initialize the primary message handler for establishing the tunnel port self.onmessage = (connEstablishedEvt) => { if (connEstablishedEvt.data === "port") { - self.tunnelPort = connEstablishedEvt.ports[0]; + self.tunnelPort = connEstablishedEvt.ports[0]; // Assign received port to tunnelPort - // as we use the reference to the MessagePort here - // the callback assignment will last as long as the MessagePort - // so we can use it to communicate with the content script + // Define what happens when a message is received through the tunnel self.tunnelPort.onmessage = (messageEvent) => { - console.log("onMessage (message-channel)", messageEvent, 'listeners', tunnelListeners); + // Notify all registered listeners with the received message const listenerCallbacks = Object.values(tunnelListeners); - if (listenerCallbacks.length) { - listenerCallbacks - .filter((cb) => !!cb) - .forEach((cb) => cb(messageEvent)); - } + listenerCallbacks.forEach((cb) => cb?.(messageEvent)); + }; + + // Handle unexpected errors in message transmission + self.tunnelPort.onmessageerror = () => { + console.error("Tunnel port encountered an error"); + self.tunnelPort = null; // Reset tunnelPort to allow reinitialization }; - // initial ack/resolve, as we were receiving the port via the tunnel script - // and it needs to be passed back to the content script, for the last step's - // Promise to resolve + // Send an acknowledgment message back to the content script after establishing the port self.tunnelPort.postMessage(null); } - // in case the caller needs to handle the message event as well (optional) + // Optionally, handle messages in the service worker if a handler is provided if (typeof onMessageHandler === "function") { - return onMessageHandler.call(self, connEstablishedEvt); + onMessageHandler.call(self, connEstablishedEvt); } }; } return { - postMessage: (message: T) => self.tunnelPort.postMessage(message), + /** + * Function to send a message through the tunnel port to the content script. + * @param message The message payload to send. + */ + postMessage: (message: T) => { + if (self.tunnelPort) { + self.tunnelPort.postMessage(message); + } else { + console.error("Tunnel port is not connected"); + } + }, + /** + * Adds a listener to handle messages received through the tunnel. + * @param listener The function to call when a message is received. + * @returns A unique key for the listener, allowing it to be removed later. + */ addListener: (listener: (e: MessageEvent) => void) => { - const listenerSecret = Math.random().toString(36); + const listenerSecret = Math.random().toString(36); // Generate a unique key for the listener tunnelListeners[listenerSecret] = listener; return listenerSecret; }, + /** + * Removes a previously added listener using its unique key. + * @param listenerSecret The unique key associated with the listener. + */ removeListener: (listenerSecret: string) => { delete tunnelListeners[listenerSecret]; }, diff --git a/src/worker.ts b/src/worker.ts index 73019e3..464eb7e 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -47,6 +47,16 @@ import { transcribeDeepgram } from "./lib/worker/transcription/deepgram"; // establish fast, MessageChannel-based communication between content script and worker const { postMessage, addListener } = useMessageChannel(); +// Set up a periodic alarm to keep the service worker alive +chrome.alarms.create("keepAlive", { periodInMinutes: 1 }); //every minutes is a safe interval + +// Listen for the alarm event to prevent the service worker from going inactive +chrome.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === "keepAlive") { + console.log("keep-alive; don't remove; otherwise the service worker will be terminated and MessageChannels will be closed"); + } +}); + /* @@ -149,68 +159,78 @@ async function decodeAudioUsingWebCodecs(file: File, codec: string, metaData: Au } */ - let offscreenClient: Client | null = null; +let messageChannel: MessageChannel | null = null; // Persistent MessageChannel + + // Create the offscreen document if it doesn't already exist async function initializeOffscreenClient() { console.log("initializeOffscreenClient"); const url = chrome.runtime.getURL('audio-processor.html'); - // Send a "ping" to the offscreenClient and check for "pong" response + // Check for an existing offscreen client and a "pong" response if (offscreenClient) { - - await new Promise((resolve) => { - - const messageChannel = new MessageChannel(); - const pingTimeout = setTimeout(async () => { + const pingTimeout = new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { console.log("Ping not answered, re-initializing offscreen client"); - offscreenClient = null; + offscreenClient = null; // Reset the client if ping fails resolve(); - }, 10); - - messageChannel.port1.onmessage = (event) => { - if (event.data === "pong") { - clearTimeout(pingTimeout); - console.log("Received pong from offscreen client"); - resolve(); - } - }; + }, 100); // Timeout duration; adjust as needed + + if (messageChannel) { + messageChannel.port1.onmessage = (event) => { + if (event.data === "pong") { + clearTimeout(timeoutId); + console.log("Received pong from offscreen client"); + resolve(); + } + }; + } else { + reject(new Error("MessageChannel not initialized")); + } if (offscreenClient) { - offscreenClient.postMessage("ping", [messageChannel.port2]); + offscreenClient.postMessage("ping", [messageChannel!.port2]); + } else { + reject(new Error("Offscreen client not initialized")); } - }) + }); + + await pingTimeout; } if (!offscreenClient) { - console.log("create offscreen client as it doesn't exist", url); + console.log("Creating new offscreen client", url); await chrome.offscreen.createDocument({ url, reasons: ["AUDIO_PLAYBACK"] as Array, - justification: 'Transcoding and slicing transcription audio.' // details for using the API + justification: "Transcoding and slicing transcription audio.", }); offscreenClient = (await self.clients.matchAll({ includeUncontrolled: true })) .find(c => c.url === url) || null; - console.log("client", offscreenClient); + if (!messageChannel) { + // Initialize MessageChannel only once + messageChannel = new MessageChannel(); + } + + console.log("client initialized:", offscreenClient); } } +// Function to send a message to the offscreen client async function postMessageToOffscreen(data: any): Promise { try { await initializeOffscreenClient(); - } catch(e) { + } catch (e) { console.warn("Warning: Initializing offscreen client failed", e); } return new Promise((resolve, reject) => { - if (offscreenClient) { - // Create a new MessageChannel for each communication - const messageChannel = new MessageChannel(); - - // Listen for the response from the offscreen client + if (offscreenClient && messageChannel) { + // Reuse the same port to send and receive data messageChannel.port1.onmessage = (event) => { if (event.data) { resolve(event.data); @@ -219,7 +239,6 @@ async function postMessageToOffscreen(data: any): Promise { } }; - // Send the message with the new MessagePort try { offscreenClient.postMessage(data, [messageChannel.port2]); } catch (error) { @@ -231,7 +250,7 @@ async function postMessageToOffscreen(data: any): Promise { }); } - +// Usage example for sending data to the offscreen client async function decodeSliceOffscreen(audioFile: File, waitingSpeechBlob: Blob) { try { console.log("decodeSliceOffscreen", audioFile);