diff --git a/screenpipe-js/browser-sdk/bun.lockb b/screenpipe-js/browser-sdk/bun.lockb index 09bb18ef7..880d948e1 100755 Binary files a/screenpipe-js/browser-sdk/bun.lockb and b/screenpipe-js/browser-sdk/bun.lockb differ diff --git a/screenpipe-js/browser-sdk/package.json b/screenpipe-js/browser-sdk/package.json index 55805b687..5c3d09c54 100644 --- a/screenpipe-js/browser-sdk/package.json +++ b/screenpipe-js/browser-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@screenpipe/browser", - "version": "0.1.15", + "version": "0.1.17", "type": "module", "main": "./dist/index.cjs", "module": "./dist/index.js", @@ -26,7 +26,8 @@ "@types/uuid": "^10.0.0", "eventsource": "^3.0.2", "express": "^4.21.1", - "node-cron": "^3.0.3" + "node-cron": "^3.0.3", + "posthog-js": "^1.205.0" }, "peerDependencies": { "typescript": "^5.0.0" diff --git a/screenpipe-js/browser-sdk/src/index.ts b/screenpipe-js/browser-sdk/src/index.ts index eb95cd00f..cbe479312 100644 --- a/screenpipe-js/browser-sdk/src/index.ts +++ b/screenpipe-js/browser-sdk/src/index.ts @@ -10,6 +10,11 @@ import type { VisionStreamResponse, } from "../../common/types"; import { toSnakeCase, convertToCamelCase } from "../../common/utils"; +import { + captureEvent, + captureMainFeatureEvent, + identifyUser, +} from "../../common/analytics"; async function sendInputControl(action: InputAction): Promise { const apiUrl = "http://localhost:3030"; @@ -49,13 +54,41 @@ export interface BrowserPipe { streamVision( includeImages?: boolean ): AsyncGenerator; + captureEvent: ( + event: string, + properties?: Record + ) => Promise; + captureMainFeatureEvent: ( + name: string, + properties?: Record + ) => Promise; } -// Browser-only implementations -export const pipe: BrowserPipe = { +class BrowserPipeImpl implements BrowserPipe { + private analyticsInitialized = false; + private analyticsEnabled = false; + private userId?: string; + private userProperties?: Record; + + private async initAnalyticsIfNeeded() { + if (this.analyticsInitialized || !this.userId) return; + + try { + const settings = { analyticsEnabled: false }; // TODO: impl settings browser side somehow ... + this.analyticsEnabled = settings.analyticsEnabled; + if (settings.analyticsEnabled) { + await identifyUser(this.userId, this.userProperties); + this.analyticsInitialized = true; + } + } catch (error) { + console.error("failed to fetch settings:", error); + } + } + async sendDesktopNotification( options: NotificationOptions ): Promise { + await this.initAnalyticsIfNeeded(); const notificationApiUrl = "http://localhost:11435"; try { await fetch(`${notificationApiUrl}/notify`, { @@ -63,16 +96,21 @@ export const pipe: BrowserPipe = { headers: { "Content-Type": "application/json" }, body: JSON.stringify(options), }); + await this.captureEvent("notification_sent", { success: true }); return true; } catch (error) { - console.error("failed to send notification:", error); + await this.captureEvent("error_occurred", { + feature: "notification", + error: "send_failed", + }); return false; } - }, + } async queryScreenpipe( params: ScreenpipeQueryParams ): Promise { + await this.initAnalyticsIfNeeded(); const queryParams = new URLSearchParams(); Object.entries(params).forEach(([key, value]) => { if (value !== undefined && value !== "") { @@ -108,21 +146,34 @@ export const pipe: BrowserPipe = { throw new Error(`http error! status: ${response.status}`); } const data = await response.json(); + await captureEvent("search_performed", { + content_type: params.contentType, + result_count: data.pagination.total, + }); return convertToCamelCase(data) as ScreenpipeResponse; } catch (error) { + await captureEvent("error_occurred", { + feature: "search", + error: "query_failed", + }); console.error("error querying screenpipe:", error); return null; } - }, + } input: { + type: (text: string) => Promise; + press: (key: string) => Promise; + moveMouse: (x: number, y: number) => Promise; + click: (button: "left" | "right" | "middle") => Promise; + } = { type: (text: string) => sendInputControl({ type: "WriteText", data: text }), press: (key: string) => sendInputControl({ type: "KeyPress", data: key }), moveMouse: (x: number, y: number) => sendInputControl({ type: "MouseMove", data: { x, y } }), click: (button: "left" | "right" | "middle") => sendInputControl({ type: "MouseClick", data: button }), - }, + }; async *streamTranscriptions(): AsyncGenerator< TranscriptionStreamResponse, @@ -134,6 +185,10 @@ export const pipe: BrowserPipe = { ); try { + await this.captureEvent("stream_started", { + feature: "transcription", + }); + while (true) { const chunk: TranscriptionChunk = await new Promise( (resolve, reject) => { @@ -169,9 +224,12 @@ export const pipe: BrowserPipe = { }; } } finally { + await this.captureEvent("stream_ended", { + feature: "transcription", + }); eventSource.close(); } - }, + } async *streamVision( includeImages: boolean = false @@ -179,8 +237,11 @@ export const pipe: BrowserPipe = { const eventSource = new EventSource( `http://localhost:3030/sse/vision?images=${includeImages}` ); - try { + await this.captureEvent("stream_started", { + feature: "vision", + }); + while (true) { const event: VisionEvent = await new Promise((resolve, reject) => { eventSource.onmessage = (event) => { @@ -197,19 +258,33 @@ export const pipe: BrowserPipe = { }; } } finally { + await this.captureEvent("stream_ended", { + feature: "vision", + }); eventSource.close(); } - }, -}; - -const sendDesktopNotification = pipe.sendDesktopNotification; -const queryScreenpipe = pipe.queryScreenpipe; -const input = pipe.input; - -export { sendDesktopNotification, queryScreenpipe, input }; -export { - toCamelCase, - toSnakeCase, - convertToCamelCase, -} from "../../common/utils"; + } + + public async captureEvent( + eventName: string, + properties?: Record + ) { + if (!this.analyticsEnabled) return; + await this.initAnalyticsIfNeeded(); + return captureEvent(eventName, properties); + } + + public async captureMainFeatureEvent( + featureName: string, + properties?: Record + ) { + if (!this.analyticsEnabled) return; + await this.initAnalyticsIfNeeded(); + return captureMainFeatureEvent(featureName, properties); + } +} + +const pipeImpl = new BrowserPipeImpl(); +export const pipe = pipeImpl; + export * from "../../common/types"; diff --git a/screenpipe-js/common/analytics.ts b/screenpipe-js/common/analytics.ts new file mode 100644 index 000000000..bd99165dd --- /dev/null +++ b/screenpipe-js/common/analytics.ts @@ -0,0 +1,40 @@ +import posthog from "posthog-js"; + +const POSTHOG_KEY = "phc_Bt8GoTBPgkCpDrbaIZzJIEYt0CrJjhBiuLaBck1clce"; +const POSTHOG_HOST = "https://eu.i.posthog.com"; + +let initialized = false; + +function initPosthog() { + if (!initialized) { + posthog.init(POSTHOG_KEY, { api_host: POSTHOG_HOST }); + initialized = true; + } +} + +export async function identifyUser( + userId: string, + properties?: Record +): Promise { + initPosthog(); + posthog.identify(userId, properties); +} + +export async function captureEvent( + name: string, + properties?: Record +): Promise { + initPosthog(); + posthog.capture(name, properties); +} + +export async function captureMainFeatureEvent( + name: string, + properties?: Record +): Promise { + initPosthog(); + posthog.capture(name, { + feature: "main", + ...properties, + }); +} diff --git a/screenpipe-js/common/types.ts b/screenpipe-js/common/types.ts index a459e8ffb..2921252dd 100644 --- a/screenpipe-js/common/types.ts +++ b/screenpipe-js/common/types.ts @@ -206,6 +206,7 @@ export interface Settings { enableFrameCache: boolean; enableUiMonitoring: boolean; aiMaxContextChars: number; + analyticsEnabled: boolean; user: User; customSettings?: Record; } diff --git a/screenpipe-js/examples/basic-transcription/.gitignore b/screenpipe-js/examples/basic-transcription/.gitignore new file mode 100644 index 000000000..9b1ee42e8 --- /dev/null +++ b/screenpipe-js/examples/basic-transcription/.gitignore @@ -0,0 +1,175 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/screenpipe-js/examples/basic-transcription/README.md b/screenpipe-js/examples/basic-transcription/README.md new file mode 100644 index 000000000..d077e874d --- /dev/null +++ b/screenpipe-js/examples/basic-transcription/README.md @@ -0,0 +1,15 @@ +# basic-transcription + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.1.38. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/screenpipe-js/examples/basic-transcription/bun.lockb b/screenpipe-js/examples/basic-transcription/bun.lockb new file mode 100755 index 000000000..01fe23bb5 Binary files /dev/null and b/screenpipe-js/examples/basic-transcription/bun.lockb differ diff --git a/screenpipe-js/examples/basic-transcription/index.ts b/screenpipe-js/examples/basic-transcription/index.ts new file mode 100644 index 000000000..1055389c0 --- /dev/null +++ b/screenpipe-js/examples/basic-transcription/index.ts @@ -0,0 +1,18 @@ +import { pipe } from "@screenpipe/js"; + +async function monitorTranscriptions() { + console.log("starting transcription monitor..."); + console.log( + "please watch this video: https://youtu.be/UF8uR6Z6KLc?t=180" + ); + + for await (const chunk of pipe.streamTranscriptions()) { + const text = chunk.choices[0].text; + const isFinal = chunk.choices[0].finish_reason === "stop"; + const device = chunk.metadata?.device; + + console.log(`[${device}] ${isFinal ? "final:" : "partial:"} ${text}`); + } +} + +monitorTranscriptions().catch(console.error); diff --git a/screenpipe-js/examples/basic-transcription/package.json b/screenpipe-js/examples/basic-transcription/package.json new file mode 100644 index 000000000..31affccc0 --- /dev/null +++ b/screenpipe-js/examples/basic-transcription/package.json @@ -0,0 +1,18 @@ +{ + "name": "basic-transcription", + "module": "index.ts", + "type": "module", + "scripts": { + "start": "bun run index.ts" + }, + "dependencies": { + "@screenpipe/js": "workspace:*", + "@screenpipe/browser": "workspace:*" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.0.0" + } +} diff --git a/screenpipe-js/examples/basic-transcription/tsconfig.json b/screenpipe-js/examples/basic-transcription/tsconfig.json new file mode 100644 index 000000000..238655f2c --- /dev/null +++ b/screenpipe-js/examples/basic-transcription/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +} diff --git a/screenpipe-js/examples/capture-main-feature/.gitignore b/screenpipe-js/examples/capture-main-feature/.gitignore new file mode 100644 index 000000000..9b1ee42e8 --- /dev/null +++ b/screenpipe-js/examples/capture-main-feature/.gitignore @@ -0,0 +1,175 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/screenpipe-js/examples/capture-main-feature/README.md b/screenpipe-js/examples/capture-main-feature/README.md new file mode 100644 index 000000000..d077e874d --- /dev/null +++ b/screenpipe-js/examples/capture-main-feature/README.md @@ -0,0 +1,15 @@ +# basic-transcription + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.1.38. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/screenpipe-js/examples/capture-main-feature/bun.lockb b/screenpipe-js/examples/capture-main-feature/bun.lockb new file mode 100755 index 000000000..01fe23bb5 Binary files /dev/null and b/screenpipe-js/examples/capture-main-feature/bun.lockb differ diff --git a/screenpipe-js/examples/capture-main-feature/index.ts b/screenpipe-js/examples/capture-main-feature/index.ts new file mode 100644 index 000000000..c4842f00b --- /dev/null +++ b/screenpipe-js/examples/capture-main-feature/index.ts @@ -0,0 +1,15 @@ +import { pipe } from "@screenpipe/js"; + +async function startScreenRecorder() { + console.log("let's send events when our main feature is used ..."); + + await pipe.captureEvent("less_useful_feature", { + dog: "woof", + }); + + await pipe.captureMainFeatureEvent("very_useful_feature", { + cat: "meow", + }); +} + +startScreenRecorder().catch(console.error); diff --git a/screenpipe-js/examples/capture-main-feature/package.json b/screenpipe-js/examples/capture-main-feature/package.json new file mode 100644 index 000000000..a8a7d912a --- /dev/null +++ b/screenpipe-js/examples/capture-main-feature/package.json @@ -0,0 +1,18 @@ +{ + "name": "capture-main-feature", + "module": "index.ts", + "type": "module", + "scripts": { + "start": "bun run index.ts" + }, + "dependencies": { + "@screenpipe/js": "workspace:*", + "@screenpipe/browser": "workspace:*" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.0.0" + } +} diff --git a/screenpipe-js/examples/capture-main-feature/tsconfig.json b/screenpipe-js/examples/capture-main-feature/tsconfig.json new file mode 100644 index 000000000..238655f2c --- /dev/null +++ b/screenpipe-js/examples/capture-main-feature/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +} diff --git a/screenpipe-js/examples/query-screenpipe/.gitignore b/screenpipe-js/examples/query-screenpipe/.gitignore new file mode 100644 index 000000000..9b1ee42e8 --- /dev/null +++ b/screenpipe-js/examples/query-screenpipe/.gitignore @@ -0,0 +1,175 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/screenpipe-js/examples/query-screenpipe/README.md b/screenpipe-js/examples/query-screenpipe/README.md new file mode 100644 index 000000000..d077e874d --- /dev/null +++ b/screenpipe-js/examples/query-screenpipe/README.md @@ -0,0 +1,15 @@ +# basic-transcription + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.1.38. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/screenpipe-js/examples/query-screenpipe/bun.lockb b/screenpipe-js/examples/query-screenpipe/bun.lockb new file mode 100755 index 000000000..01fe23bb5 Binary files /dev/null and b/screenpipe-js/examples/query-screenpipe/bun.lockb differ diff --git a/screenpipe-js/examples/query-screenpipe/index.ts b/screenpipe-js/examples/query-screenpipe/index.ts new file mode 100644 index 000000000..f3c92b4ee --- /dev/null +++ b/screenpipe-js/examples/query-screenpipe/index.ts @@ -0,0 +1,54 @@ +import { pipe } from "@screenpipe/js"; + +async function queryScreenpipe() { + console.log("starting query screenpipe..."); + console.log("------------------------------"); + console.log("querying last 5 minutes of activity..."); + console.log("------------------------------"); + + // get content from last 5 minutes + const fiveMinutesAgo = new Date(Date.now() - 5 * 60 * 1000).toISOString(); + + const results = await pipe.queryScreenpipe({ + startTime: fiveMinutesAgo, + limit: 10, + contentType: "all", // can be "text", "vision", or "all" + }); + + if (!results) { + console.log("no results found or error occurred"); + return; + } + + console.log(`found ${results.pagination.total} items`); + + // process each result + for (const item of results.data) { + console.log("\n--- new item ---"); + console.log(`type: ${item.type}`); + console.log(`timestamp: ${item.content.timestamp}`); + + if (item.type === "OCR") { + console.log(`OCR: ${JSON.stringify(item.content)}`); + } else if (item.type === "Audio") { + console.log(`transcript: ${JSON.stringify(item.content)}`); + } else if (item.type === "UI") { + console.log(`UI: ${JSON.stringify(item.content)}`); + } + + // here you could send to openai or other ai service + // example pseudo-code: + // const aiResponse = await openai.chat.completions.create({ + // messages: [{ role: "user", content: item.content }], + // model: "gpt-4" + // }); + console.log( + "\n\nnow you could send to openai or other ai service with this code:\n" + ); + console.log( + "const aiResponse = await openai.chat.completions.create({ messages: [{ role: 'user', content: item.content }], model: 'gpt-4' });" + ); + } +} + +queryScreenpipe().catch(console.error); diff --git a/screenpipe-js/examples/query-screenpipe/package.json b/screenpipe-js/examples/query-screenpipe/package.json new file mode 100644 index 000000000..e49040928 --- /dev/null +++ b/screenpipe-js/examples/query-screenpipe/package.json @@ -0,0 +1,18 @@ +{ + "name": "query-screenpipe", + "module": "index.ts", + "type": "module", + "scripts": { + "start": "bun run index.ts" + }, + "dependencies": { + "@screenpipe/js": "workspace:*", + "@screenpipe/browser": "workspace:*" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.0.0" + } +} diff --git a/screenpipe-js/examples/query-screenpipe/tsconfig.json b/screenpipe-js/examples/query-screenpipe/tsconfig.json new file mode 100644 index 000000000..238655f2c --- /dev/null +++ b/screenpipe-js/examples/query-screenpipe/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +} diff --git a/screenpipe-js/examples/vision-monitor/.gitignore b/screenpipe-js/examples/vision-monitor/.gitignore new file mode 100644 index 000000000..9b1ee42e8 --- /dev/null +++ b/screenpipe-js/examples/vision-monitor/.gitignore @@ -0,0 +1,175 @@ +# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore + +# Logs + +logs +_.log +npm-debug.log_ +yarn-debug.log* +yarn-error.log* +lerna-debug.log* +.pnpm-debug.log* + +# Caches + +.cache + +# Diagnostic reports (https://nodejs.org/api/report.html) + +report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json + +# Runtime data + +pids +_.pid +_.seed +*.pid.lock + +# Directory for instrumented libs generated by jscoverage/JSCover + +lib-cov + +# Coverage directory used by tools like istanbul + +coverage +*.lcov + +# nyc test coverage + +.nyc_output + +# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) + +.grunt + +# Bower dependency directory (https://bower.io/) + +bower_components + +# node-waf configuration + +.lock-wscript + +# Compiled binary addons (https://nodejs.org/api/addons.html) + +build/Release + +# Dependency directories + +node_modules/ +jspm_packages/ + +# Snowpack dependency directory (https://snowpack.dev/) + +web_modules/ + +# TypeScript cache + +*.tsbuildinfo + +# Optional npm cache directory + +.npm + +# Optional eslint cache + +.eslintcache + +# Optional stylelint cache + +.stylelintcache + +# Microbundle cache + +.rpt2_cache/ +.rts2_cache_cjs/ +.rts2_cache_es/ +.rts2_cache_umd/ + +# Optional REPL history + +.node_repl_history + +# Output of 'npm pack' + +*.tgz + +# Yarn Integrity file + +.yarn-integrity + +# dotenv environment variable files + +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# parcel-bundler cache (https://parceljs.org/) + +.parcel-cache + +# Next.js build output + +.next +out + +# Nuxt.js build / generate output + +.nuxt +dist + +# Gatsby files + +# Comment in the public line in if your project uses Gatsby and not Next.js + +# https://nextjs.org/blog/next-9-1#public-directory-support + +# public + +# vuepress build output + +.vuepress/dist + +# vuepress v2.x temp and cache directory + +.temp + +# Docusaurus cache and generated files + +.docusaurus + +# Serverless directories + +.serverless/ + +# FuseBox cache + +.fusebox/ + +# DynamoDB Local files + +.dynamodb/ + +# TernJS port file + +.tern-port + +# Stores VSCode versions used for testing VSCode extensions + +.vscode-test + +# yarn v2 + +.yarn/cache +.yarn/unplugged +.yarn/build-state.yml +.yarn/install-state.gz +.pnp.* + +# IntelliJ based IDEs +.idea + +# Finder (MacOS) folder config +.DS_Store diff --git a/screenpipe-js/examples/vision-monitor/README.md b/screenpipe-js/examples/vision-monitor/README.md new file mode 100644 index 000000000..d077e874d --- /dev/null +++ b/screenpipe-js/examples/vision-monitor/README.md @@ -0,0 +1,15 @@ +# basic-transcription + +To install dependencies: + +```bash +bun install +``` + +To run: + +```bash +bun run index.ts +``` + +This project was created using `bun init` in bun v1.1.38. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime. diff --git a/screenpipe-js/examples/vision-monitor/bun.lockb b/screenpipe-js/examples/vision-monitor/bun.lockb new file mode 100755 index 000000000..01fe23bb5 Binary files /dev/null and b/screenpipe-js/examples/vision-monitor/bun.lockb differ diff --git a/screenpipe-js/examples/vision-monitor/index.ts b/screenpipe-js/examples/vision-monitor/index.ts new file mode 100644 index 000000000..89521f543 --- /dev/null +++ b/screenpipe-js/examples/vision-monitor/index.ts @@ -0,0 +1,73 @@ +import { pipe } from "@screenpipe/js"; +import * as fs from "fs/promises"; + +async function monitorVision() { + console.log("starting vision monitor..."); + console.log("------------------------------"); + console.log("to view screenshots:"); + console.log( + "1. paste this in a new terminal: 'open /Users/louisbeaumont/Documents/screenpipe/screenpipe-js/examples/vision-monitor/screenshots/viewer.html'" + ); + console.log("2. watch live updates every 1s"); + console.log("------------------------------"); + + // create screenshots directory + await fs.mkdir("screenshots", { recursive: true }); + + // create simple html viewer + const htmlContent = ` + + + + screenpipe vision monitor + + + + +
screenpipe vision monitor
+ + + + `; + await fs.writeFile("screenshots/viewer.html", htmlContent); + + for await (const event of pipe.streamVision(true)) { + const { timestamp, window_name, image } = event.data; + + if (image) { + const filename = `screenshots/${timestamp}-${window_name}.png`; + // save to archive + await fs.writeFile(filename, Buffer.from(image, "base64")); + // update latest for viewer + await fs.writeFile( + "screenshots/latest.png", + Buffer.from(image, "base64") + ); + console.log(`saved screenshot: ${filename}`); + } + + console.log(`window: ${window_name}`); + } +} + +monitorVision().catch(console.error); diff --git a/screenpipe-js/examples/vision-monitor/package.json b/screenpipe-js/examples/vision-monitor/package.json new file mode 100644 index 000000000..039869c23 --- /dev/null +++ b/screenpipe-js/examples/vision-monitor/package.json @@ -0,0 +1,18 @@ +{ + "name": "vision-monitor", + "module": "index.ts", + "type": "module", + "scripts": { + "start": "bun run index.ts" + }, + "dependencies": { + "@screenpipe/js": "workspace:*", + "@screenpipe/browser": "workspace:*" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "peerDependencies": { + "typescript": "^5.0.0" + } +} diff --git a/screenpipe-js/examples/vision-monitor/tsconfig.json b/screenpipe-js/examples/vision-monitor/tsconfig.json new file mode 100644 index 000000000..238655f2c --- /dev/null +++ b/screenpipe-js/examples/vision-monitor/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + // Enable latest features + "lib": ["ESNext", "DOM"], + "target": "ESNext", + "module": "ESNext", + "moduleDetection": "force", + "jsx": "react-jsx", + "allowJs": true, + + // Bundler mode + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + + // Best practices + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + + // Some stricter flags (disabled by default) + "noUnusedLocals": false, + "noUnusedParameters": false, + "noPropertyAccessFromIndexSignature": false + } +} diff --git a/screenpipe-js/node-sdk/bun.lockb b/screenpipe-js/node-sdk/bun.lockb index 21d5db97e..77b8a0388 100755 Binary files a/screenpipe-js/node-sdk/bun.lockb and b/screenpipe-js/node-sdk/bun.lockb differ diff --git a/screenpipe-js/node-sdk/package.json b/screenpipe-js/node-sdk/package.json index 03307c92f..fd62f6546 100644 --- a/screenpipe-js/node-sdk/package.json +++ b/screenpipe-js/node-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@screenpipe/js", - "version": "1.0.4", + "version": "1.0.6", "type": "module", "main": "./dist/index.cjs", "module": "./dist/index.js", @@ -26,7 +26,8 @@ "@types/uuid": "^10.0.0", "eventsource": "^3.0.2", "express": "^4.21.1", - "node-cron": "^3.0.3" + "node-cron": "^3.0.3", + "posthog-js": "^1.205.0" }, "peerDependencies": { "typescript": "^5.0.0" diff --git a/screenpipe-js/node-sdk/src/Scheduler.ts b/screenpipe-js/node-sdk/src/Scheduler.ts deleted file mode 100644 index ebfd7a22f..000000000 --- a/screenpipe-js/node-sdk/src/Scheduler.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Task } from "./Task"; -import cron from "node-cron"; - -export class Scheduler { - private tasks: Task[] = []; - - constructor() {} - - task(name: string): Task { - const task = new Task(name); - this.tasks.push(task); - return task; - } - - start() { - this.tasks.forEach((task) => task.schedule()); - } - - async stop(): Promise { - cron.getTasks().forEach((task: any) => task.stop()); - - this.tasks = []; - } -} diff --git a/screenpipe-js/node-sdk/src/SettingsManger.ts b/screenpipe-js/node-sdk/src/SettingsManager.ts similarity index 99% rename from screenpipe-js/node-sdk/src/SettingsManger.ts rename to screenpipe-js/node-sdk/src/SettingsManager.ts index e3015c46d..09886c536 100644 --- a/screenpipe-js/node-sdk/src/SettingsManger.ts +++ b/screenpipe-js/node-sdk/src/SettingsManager.ts @@ -27,8 +27,9 @@ const DEFAULT_SETTINGS: Settings = { }, enableFrameCache: true, enableUiMonitoring: false, - aiMaxContextChars: 128000, + aiMaxContextChars: 512000, user: {}, + analyticsEnabled: true, }; export class SettingsManager { diff --git a/screenpipe-js/node-sdk/src/index.ts b/screenpipe-js/node-sdk/src/index.ts index eb33e9970..2f3b17b7d 100644 --- a/screenpipe-js/node-sdk/src/index.ts +++ b/screenpipe-js/node-sdk/src/index.ts @@ -1,8 +1,4 @@ -import fs from "fs/promises"; -import path from "path"; import type { - PipeConfig, - ParsedConfig, InputAction, InputControlResponse, ScreenpipeQueryParams, @@ -11,18 +7,24 @@ import type { TranscriptionChunk, VisionEvent, VisionStreamResponse, + NotificationOptions, } from "../../common/types"; -import { - toSnakeCase, - convertToCamelCase, - toCamelCase, -} from "../../common/utils"; -import { SettingsManager } from "./SettingsManger"; -import { Scheduler } from "./Scheduler"; +import { toSnakeCase, convertToCamelCase } from "../../common/utils"; +import { SettingsManager } from "./SettingsManager"; import { InboxManager } from "./InboxManager"; import { EventSource } from "eventsource"; +import { + captureEvent, + captureMainFeatureEvent, + identifyUser, +} from "../../common/analytics"; class NodePipe { + private analyticsInitialized = false; + private analyticsEnabled = true; + private userId?: string; + private userProperties?: Record; + public input = { type: (text: string) => this.sendInputControl({ type: "WriteText", data: text }), @@ -35,12 +37,12 @@ class NodePipe { }; public settings = new SettingsManager(); - public scheduler = new Scheduler(); public inbox = new InboxManager(); public async sendDesktopNotification( options: NotificationOptions ): Promise { + await this.initAnalyticsIfNeeded(); const notificationApiUrl = "http://localhost:11435"; try { await fetch(`${notificationApiUrl}/notify`, { @@ -48,14 +50,22 @@ class NodePipe { headers: { "Content-Type": "application/json" }, body: JSON.stringify(options), }); + await captureEvent("notification_sent", { + success: true, + }); return true; } catch (error) { + await captureEvent("error_occurred", { + feature: "notification", + error: "send_failed", + }); console.error("failed to send notification:", error); return false; } } public async sendInputControl(action: InputAction): Promise { + await this.initAnalyticsIfNeeded(); const apiUrl = process.env.SCREENPIPE_SERVER_URL || "http://localhost:3030"; try { const response = await fetch(`${apiUrl}/experimental/input_control`, { @@ -77,6 +87,7 @@ class NodePipe { public async queryScreenpipe( params: ScreenpipeQueryParams ): Promise { + await this.initAnalyticsIfNeeded(); const queryParams = new URLSearchParams(); Object.entries(params).forEach(([key, value]) => { if (value !== undefined && value !== "") { @@ -112,33 +123,21 @@ class NodePipe { throw new Error(`http error! status: ${response.status}`); } const data = await response.json(); + await captureEvent("search_performed", { + content_type: params.contentType, + result_count: data.pagination.total, + }); return convertToCamelCase(data) as ScreenpipeResponse; } catch (error) { + await captureEvent("error_occurred", { + feature: "search", + error: "query_failed", + }); console.error("error querying screenpipe:", error); return null; } } - public async loadPipeConfig(): Promise { - try { - const baseDir = process.env.SCREENPIPE_DIR || process.cwd(); - const pipeId = process.env.PIPE_ID || path.basename(process.cwd()); - const configPath = `${baseDir}/pipes/${pipeId}/pipe.json`; - - const configContent = await fs.readFile(configPath, "utf8"); - const parsedConfig: ParsedConfig = JSON.parse(configContent); - const config: PipeConfig = {}; - parsedConfig.fields.forEach((field) => { - config[field.name] = - field.value !== undefined ? field.value : field.default; - }); - return config; - } catch (error) { - console.error("error loading pipe.json:", error); - return {}; - } - } - public async *streamTranscriptions(): AsyncGenerator< TranscriptionStreamResponse, void, @@ -148,6 +147,10 @@ class NodePipe { const eventSource = new EventSource(`${apiUrl}/sse/transcriptions`); try { + await captureEvent("stream_started", { + feature: "transcription", + }); + while (true) { const chunk: TranscriptionChunk = await new Promise( (resolve, reject) => { @@ -183,6 +186,9 @@ class NodePipe { }; } } finally { + await captureEvent("stream_ended", { + feature: "transcription", + }); eventSource.close(); } } @@ -215,10 +221,39 @@ class NodePipe { eventSource.close(); } } + + private async initAnalyticsIfNeeded() { + if (this.analyticsInitialized || !this.userId) return; + + const settings = await this.settings.getAll(); + this.analyticsEnabled = settings.analyticsEnabled; + if (settings.analyticsEnabled) { + await identifyUser(this.userId, this.userProperties); + this.analyticsInitialized = true; + } + } + + public async captureEvent( + eventName: string, + properties?: Record + ) { + if (!this.analyticsEnabled) return; + await this.initAnalyticsIfNeeded(); + return captureEvent(eventName, properties); + } + + public async captureMainFeatureEvent( + featureName: string, + properties?: Record + ) { + if (!this.analyticsEnabled) return; + await this.initAnalyticsIfNeeded(); + return captureMainFeatureEvent(featureName, properties); + } } const pipe = new NodePipe(); -export { pipe, toCamelCase, toSnakeCase, convertToCamelCase }; +export { pipe }; export * from "../../common/types"; diff --git a/screenpipe-js/node-sdk/tests/loadPipeConfig.test.ts b/screenpipe-js/node-sdk/tests/loadPipeConfig.test.ts deleted file mode 100644 index 0e6303522..000000000 --- a/screenpipe-js/node-sdk/tests/loadPipeConfig.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, test, beforeEach } from "bun:test"; -import { v4 as uuid } from "uuid"; -import { pipe } from "../src/index"; -import { mkdir, writeFile } from "node:fs/promises"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; -import type { ParsedConfig } from "../../common/types"; - -describe("loadPipeConfig", () => { - let pipeDir: string; - let pipeId: string; - - const setupPipeDir = async (config: ParsedConfig) => { - pipeId = uuid(); - pipeDir = join(tmpdir(), `screenpipe-test-${uuid()}`); - - process.env.SCREENPIPE_DIR = pipeDir; - process.env.PIPE_ID = pipeId; - - const pipePath = join(pipeDir, "pipes", pipeId); - await mkdir(pipePath, { recursive: true }); - await writeFile(join(pipePath, "pipe.json"), JSON.stringify(config)); - }; - - const generateConfig = () => ({ - interval: Math.floor(Math.random() * 100), - summaryFrequency: Math.floor(Math.random() * 100), - emailAddress: `mail+${uuid()}@contact.com`, - }); - - test("should return empty object if pipe.json is not found", async () => { - process.env.SCREENPIPE_DIR = uuid(); - process.env.PIPE_ID = uuid(); - const loadedConfig = await pipe.loadPipeConfig(); - expect(loadedConfig).toEqual({}); - }); - - test("should load config from SCREENPIPE_DIR/pipes/PIPE_ID/pipe.json", async () => { - const config = generateConfig(); - - await setupPipeDir({ - fields: [ - { name: "interval", value: config.interval }, - { name: "summaryFrequency", value: config.summaryFrequency }, - { name: "emailAddress", value: config.emailAddress }, - ], - }); - - const loadedConfig = await pipe.loadPipeConfig(); - expect(loadedConfig).toEqual(config); - }); - - test("should load default values if not provided in config", async () => { - const config = generateConfig(); - - await setupPipeDir({ - fields: [ - { name: "interval", value: config.interval }, - { name: "summaryFrequency", default: 5 }, - { name: "emailAddress", value: config.emailAddress }, - ], - }); - - const loadedConfig = await pipe.loadPipeConfig(); - expect(loadedConfig).toEqual({ ...config, summaryFrequency: 5 }); - }); -}); diff --git a/screenpipe-js/node-sdk/tests/scheduler.test.ts b/screenpipe-js/node-sdk/tests/scheduler.test.ts deleted file mode 100644 index cf7e3a565..000000000 --- a/screenpipe-js/node-sdk/tests/scheduler.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { describe, expect, test, mock, beforeEach, afterEach } from "bun:test"; -import { pipe } from "../src/index"; -import cron from "node-cron"; - -describe("Scheduler", () => { - let cronSpy: any; - let scheduledTasks: { expression: string; handler: () => Promise; options: any }[] = []; - - beforeEach(() => { - scheduledTasks = []; - cronSpy = mock((expression: string, handler: () => Promise, options: any) => { - scheduledTasks.push({ expression, handler, options }); - return { - stop: () => {}, - }; - }); - // @ts-ignore - cron.schedule = cronSpy; - }); - - afterEach(() => { - mock.restore(); - pipe.scheduler.stop(); - scheduledTasks = []; - }); - - test("Scheduler - task execution", async () => { - const scheduler = pipe.scheduler; - const mockTask = mock(() => Promise.resolve()); - - scheduler.task("testTask").every("1 second").do(mockTask); - scheduler.start(); - - expect(cronSpy).toHaveBeenCalledTimes(1); - expect(cronSpy).toHaveBeenCalledWith("*/1 * * * * *", mockTask, { - name: "testTask", - }); - - // Execute the scheduled task handler - await scheduledTasks[0].handler(); - expect(mockTask).toHaveBeenCalled(); - }); - - test("multiple tasks", async () => { - const scheduler = pipe.scheduler; - const mockTask1 = mock(() => Promise.resolve()); - const mockTask2 = mock(() => Promise.resolve()); - - scheduler.task("task1").every("1 second").do(mockTask1); - scheduler.task("task2").every("2 seconds").do(mockTask2); - scheduler.start(); - - expect(cronSpy).toHaveBeenCalledTimes(2); - expect(cronSpy).toHaveBeenNthCalledWith(1, "*/1 * * * * *", mockTask1, { - name: "task1", - }); - expect(cronSpy).toHaveBeenNthCalledWith(2, "*/2 * * * * *", mockTask2, { - name: "task2", - }); - - // Execute both task handlers - await scheduledTasks[0].handler(); - await scheduledTasks[1].handler(); - - expect(mockTask1).toHaveBeenCalled(); - expect(mockTask2).toHaveBeenCalled(); - }); -});