diff --git a/cli/Cargo.lock b/cli/Cargo.lock index 12b3053..8c68d8a 100644 --- a/cli/Cargo.lock +++ b/cli/Cargo.lock @@ -1113,7 +1113,7 @@ dependencies = [ [[package]] name = "spider-client" -version = "0.1.19" +version = "0.1.20" dependencies = [ "reqwest", "serde", @@ -1124,7 +1124,7 @@ dependencies = [ [[package]] name = "spider-cloud-cli" -version = "0.1.3" +version = "0.1.4" dependencies = [ "clap", "keyring", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index bf173b0..77e7fd8 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider-cloud-cli" -version = "0.1.3" +version = "0.1.4" edition = "2021" authors = [ "j-mendez "] description = "The Spider Cloud CLI for web crawling and scraping" diff --git a/javascript/README.md b/javascript/README.md index c18b8e7..3571343 100644 --- a/javascript/README.md +++ b/javascript/README.md @@ -148,24 +148,6 @@ spider - **`deleteData(table, params)`**: Delete records from the DB. - **`createSignedUrl(domain, params)`**: Download the records from the DB. -## Supabase - -You can use [Supabase](https://supabase.com/docs/reference/javascript) to directly connect to instances and write your own logic. First, you need to install `@supabase/supabase-js` since this package does not include the dependency by default. This keeps the bundle size small and allows for lazy imports of the client. - -```ts -const spiderClient = new Spider({ apiKey: process.env.SPIDER_API_KEY }); - -// first init the supabase client to get the anon_key from the server. -await spiderClient.init_supabase(); - -const auth = await spiderClient.supabase?.auth.signInWithPassword({ - email: process.env.SPIDER_EMAIL || "", - password: process.env.SPIDER_PASSWORD || "", -}); - -// now you can do anything with spiderClient.supabase -``` - ## Error Handling The SDK provides robust error handling and will throw exceptions when it encounters critical issues. Always use `.catch()` on promises to handle these errors gracefully. diff --git a/javascript/__tests__/spiderwebai.test.ts b/javascript/__tests__/spiderwebai.test.ts index 3f08ce5..48ce334 100644 --- a/javascript/__tests__/spiderwebai.test.ts +++ b/javascript/__tests__/spiderwebai.test.ts @@ -178,16 +178,4 @@ describe("Spider JS SDK", () => { assert(typeof signedUrl === 'string'); assert(typeof fileName === 'string'); }); - - test("should connect with supabase", async () => { - const spiderClient = new Spider(); - await spiderClient.init_supabase(); - - const auth = await spiderClient.supabase?.auth.signInWithPassword({ - email: process.env.SPIDER_EMAIL || "", - password: process.env.SPIDER_PASSWORD || "", - }); - - assert(auth); - }); }); diff --git a/javascript/package-lock.json b/javascript/package-lock.json index b222104..36939c6 100644 --- a/javascript/package-lock.json +++ b/javascript/package-lock.json @@ -1,25 +1,24 @@ { "name": "@spider-cloud/spider-client", - "version": "0.0.70", + "version": "0.0.71", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@spider-cloud/spider-client", - "version": "0.0.70", + "version": "0.0.71", "license": "MIT", "devDependencies": { - "@supabase/supabase-js": "^2.45.0", - "@types/node": "20.14.2", + "@types/node": "22.7.5", "dotenv": "^16.4.5", - "tsx": "^4.16.5", - "typescript": "5.4.5" + "tsx": "^4.19.1", + "typescript": "5.6.3" } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", + "integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==", "cpu": [ "ppc64" ], @@ -29,13 +28,13 @@ "aix" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz", + "integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==", "cpu": [ "arm" ], @@ -45,13 +44,13 @@ "android" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz", + "integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==", "cpu": [ "arm64" ], @@ -61,13 +60,13 @@ "android" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz", + "integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==", "cpu": [ "x64" ], @@ -77,13 +76,13 @@ "android" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz", + "integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==", "cpu": [ "arm64" ], @@ -93,13 +92,13 @@ "darwin" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz", + "integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==", "cpu": [ "x64" ], @@ -109,13 +108,13 @@ "darwin" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz", + "integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==", "cpu": [ "arm64" ], @@ -125,13 +124,13 @@ "freebsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz", + "integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==", "cpu": [ "x64" ], @@ -141,13 +140,13 @@ "freebsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz", + "integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==", "cpu": [ "arm" ], @@ -157,13 +156,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz", + "integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==", "cpu": [ "arm64" ], @@ -173,13 +172,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz", + "integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==", "cpu": [ "ia32" ], @@ -189,13 +188,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz", + "integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==", "cpu": [ "loong64" ], @@ -205,13 +204,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz", + "integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==", "cpu": [ "mips64el" ], @@ -221,13 +220,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz", + "integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==", "cpu": [ "ppc64" ], @@ -237,13 +236,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz", + "integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==", "cpu": [ "riscv64" ], @@ -253,13 +252,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz", + "integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==", "cpu": [ "s390x" ], @@ -269,13 +268,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz", + "integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==", "cpu": [ "x64" ], @@ -285,13 +284,13 @@ "linux" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz", + "integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==", "cpu": [ "x64" ], @@ -301,13 +300,29 @@ "netbsd" ], "engines": { - "node": ">=12" + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz", + "integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz", + "integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==", "cpu": [ "x64" ], @@ -317,13 +332,13 @@ "openbsd" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz", + "integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==", "cpu": [ "x64" ], @@ -333,13 +348,13 @@ "sunos" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz", + "integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==", "cpu": [ "arm64" ], @@ -349,13 +364,13 @@ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz", + "integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==", "cpu": [ "ia32" ], @@ -365,13 +380,13 @@ "win32" ], "engines": { - "node": ">=12" + "node": ">=18" } }, "node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz", + "integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==", "cpu": [ "x64" ], @@ -381,105 +396,16 @@ "win32" ], "engines": { - "node": ">=12" - } - }, - "node_modules/@supabase/auth-js": { - "version": "2.64.4", - "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.64.4.tgz", - "integrity": "sha512-9ITagy4WP4FLl+mke1rchapOH0RQpf++DI+WSG2sO1OFOZ0rW3cwAM0nCrMOxu+Zw4vJ4zObc08uvQrXx590Tg==", - "dev": true, - "dependencies": { - "@supabase/node-fetch": "^2.6.14" - } - }, - "node_modules/@supabase/functions-js": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.4.1.tgz", - "integrity": "sha512-8sZ2ibwHlf+WkHDUZJUXqqmPvWQ3UHN0W30behOJngVh/qHHekhJLCFbh0AjkE9/FqqXtf9eoVvmYgfCLk5tNA==", - "dev": true, - "dependencies": { - "@supabase/node-fetch": "^2.6.14" - } - }, - "node_modules/@supabase/node-fetch": { - "version": "2.6.15", - "resolved": "https://registry.npmjs.org/@supabase/node-fetch/-/node-fetch-2.6.15.tgz", - "integrity": "sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==", - "dev": true, - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - } - }, - "node_modules/@supabase/postgrest-js": { - "version": "1.15.8", - "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.15.8.tgz", - "integrity": "sha512-YunjXpoQjQ0a0/7vGAvGZA2dlMABXFdVI/8TuVKtlePxyT71sl6ERl6ay1fmIeZcqxiuFQuZw/LXUuStUG9bbg==", - "dev": true, - "dependencies": { - "@supabase/node-fetch": "^2.6.14" - } - }, - "node_modules/@supabase/realtime-js": { - "version": "2.10.2", - "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.10.2.tgz", - "integrity": "sha512-qyCQaNg90HmJstsvr2aJNxK2zgoKh9ZZA8oqb7UT2LCh3mj9zpa3Iwu167AuyNxsxrUE8eEJ2yH6wLCij4EApA==", - "dev": true, - "dependencies": { - "@supabase/node-fetch": "^2.6.14", - "@types/phoenix": "^1.5.4", - "@types/ws": "^8.5.10", - "ws": "^8.14.2" - } - }, - "node_modules/@supabase/storage-js": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.6.0.tgz", - "integrity": "sha512-REAxr7myf+3utMkI2oOmZ6sdplMZZ71/2NEIEMBZHL9Fkmm3/JnaOZVSRqvG4LStYj2v5WhCruCzuMn6oD/Drw==", - "dev": true, - "dependencies": { - "@supabase/node-fetch": "^2.6.14" - } - }, - "node_modules/@supabase/supabase-js": { - "version": "2.45.0", - "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.45.0.tgz", - "integrity": "sha512-j66Mfs8RhzCQCKxKogAFQYH9oNhRmgIdKk6pexguI2Oc7hi+nL9UNJug5aL1tKnBdaBM3h65riPLQSdL6sWa3Q==", - "dev": true, - "dependencies": { - "@supabase/auth-js": "2.64.4", - "@supabase/functions-js": "2.4.1", - "@supabase/node-fetch": "2.6.15", - "@supabase/postgrest-js": "1.15.8", - "@supabase/realtime-js": "2.10.2", - "@supabase/storage-js": "2.6.0" + "node": ">=18" } }, "node_modules/@types/node": { - "version": "20.14.2", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.2.tgz", - "integrity": "sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==", + "version": "22.7.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.5.tgz", + "integrity": "sha512-jML7s2NAzMWc//QSJ1a3prpk78cOPchGvXJsC3C6R6PSMoooztvRVQEz89gmBTBY1SPMaqo5teB4uNHPdetShQ==", "dev": true, "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/@types/phoenix": { - "version": "1.6.5", - "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.5.tgz", - "integrity": "sha512-xegpDuR+z0UqG9fwHqNoy3rI7JDlvaPh2TY47Fl80oq6g+hXT+c/LEuE43X48clZ6lOfANl5WrPur9fYO1RJ/w==", - "dev": true - }, - "node_modules/@types/ws": { - "version": "8.5.10", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", - "integrity": "sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==", - "dev": true, - "dependencies": { - "@types/node": "*" + "undici-types": "~6.19.2" } }, "node_modules/dotenv": { @@ -495,41 +421,42 @@ } }, "node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz", + "integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==", "dev": true, "hasInstallScript": true, "bin": { "esbuild": "bin/esbuild" }, "engines": { - "node": ">=12" + "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" + "@esbuild/aix-ppc64": "0.23.1", + "@esbuild/android-arm": "0.23.1", + "@esbuild/android-arm64": "0.23.1", + "@esbuild/android-x64": "0.23.1", + "@esbuild/darwin-arm64": "0.23.1", + "@esbuild/darwin-x64": "0.23.1", + "@esbuild/freebsd-arm64": "0.23.1", + "@esbuild/freebsd-x64": "0.23.1", + "@esbuild/linux-arm": "0.23.1", + "@esbuild/linux-arm64": "0.23.1", + "@esbuild/linux-ia32": "0.23.1", + "@esbuild/linux-loong64": "0.23.1", + "@esbuild/linux-mips64el": "0.23.1", + "@esbuild/linux-ppc64": "0.23.1", + "@esbuild/linux-riscv64": "0.23.1", + "@esbuild/linux-s390x": "0.23.1", + "@esbuild/linux-x64": "0.23.1", + "@esbuild/netbsd-x64": "0.23.1", + "@esbuild/openbsd-arm64": "0.23.1", + "@esbuild/openbsd-x64": "0.23.1", + "@esbuild/sunos-x64": "0.23.1", + "@esbuild/win32-arm64": "0.23.1", + "@esbuild/win32-ia32": "0.23.1", + "@esbuild/win32-x64": "0.23.1" } }, "node_modules/fsevents": { @@ -567,19 +494,13 @@ "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "dev": true - }, "node_modules/tsx": { - "version": "4.16.5", - "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.16.5.tgz", - "integrity": "sha512-ArsiAQHEW2iGaqZ8fTA1nX0a+lN5mNTyuGRRO6OW3H/Yno1y9/t1f9YOI1Cfoqz63VAthn++ZYcbDP7jPflc+A==", + "version": "4.19.1", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.1.tgz", + "integrity": "sha512-0flMz1lh74BR4wOvBjuh9olbnwqCPc35OOlfyzHba0Dc+QNUeWX/Gq2YTbnwcWPO3BMd8fkzRVrHcsR+a7z7rA==", "dev": true, "dependencies": { - "esbuild": "~0.21.5", + "esbuild": "~0.23.0", "get-tsconfig": "^4.7.5" }, "bin": { @@ -593,9 +514,9 @@ } }, "node_modules/typescript": { - "version": "5.4.5", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", - "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -606,47 +527,10 @@ } }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "dev": true - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", "dev": true - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dev": true, - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/ws": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", - "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", - "dev": true, - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } } } } diff --git a/javascript/package.json b/javascript/package.json index 403af19..85b561c 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -1,6 +1,6 @@ { "name": "@spider-cloud/spider-client", - "version": "0.0.70", + "version": "0.0.71", "description": "Isomorphic Javascript SDK for Spider Cloud services", "scripts": { "test": "node --import tsx --test __tests__/*test.ts", @@ -23,10 +23,9 @@ "author": "Jeff Mendez", "license": "MIT", "devDependencies": { - "@supabase/supabase-js": "^2.45.0", - "@types/node": "20.14.2", + "@types/node": "22.7.5", "dotenv": "^16.4.5", - "tsx": "^4.16.5", - "typescript": "5.4.5" + "tsx": "^4.19.1", + "typescript": "5.6.3" } } diff --git a/javascript/src/client.ts b/javascript/src/client.ts index faa8640..ae23112 100644 --- a/javascript/src/client.ts +++ b/javascript/src/client.ts @@ -9,7 +9,6 @@ import { ApiVersion, } from "./config"; import { version } from "../package.json"; -import { Supabase } from "./supabase"; import { streamReader } from "./utils/stream-reader"; /** @@ -43,20 +42,6 @@ export class Spider { } } - /** - * Init a supabase client. - */ - async init_supabase() { - return await Supabase.init(); - } - - /** - * The supabase client to manage data. - */ - get supabase() { - return Supabase.client; - } - /** * Internal method to handle POST requests. * @param {string} endpoint - The API endpoint to which the POST request should be sent. diff --git a/javascript/src/config.ts b/javascript/src/config.ts index 6200fe8..da77db2 100644 --- a/javascript/src/config.ts +++ b/javascript/src/config.ts @@ -48,6 +48,38 @@ interface Timeout { nanos: number; } +/** + * Represents the webhook configuration. + * @typedef {Object} WebhookSettings + * @property {Object} object - The webhook configuration. + */ +interface WebhookSettings { + /** + * The URL or endpoint where the webhook information will be sent. + */ + destination: string; + /** + * Flag to indicate an action should be taken when all credits are depleted. + */ + on_credits_depleted: boolean; + /** + * Flag to indicate an action should be taken when half of the credits are depleted. + */ + on_credits_half_depleted: boolean; + /** + * Flag to trigger a notification on a website status update event. + */ + on_website_status: boolean; + /** + * Flag to send information about a new page find, such as links and data size. + */ + on_find: boolean; + /** + * Flag to handle the metadata of a new page that has been found. + */ + on_find_metadata: boolean; +} + /** * Represents the idle network configuration. * @typedef {Object} IdleNetwork @@ -137,6 +169,15 @@ export type WebAutomation = | { Fill: { selector: string; value?: string } } | { InfiniteScroll: number }; +export type ReturnFormat = + | "markdown" + | "commonmark" + | "raw" + | "text" + | "html2text" + | "bytes" + | "empty"; + // Map automation scripts for paths or urls. export type WebAutomationMap = Record; // Map execution scripts for paths or urls. @@ -168,15 +209,9 @@ export interface SpiderParams { limit?: number; /** - * The format in which the result should be returned. + * The format in which the result should be returned. When setting the return format as an array a object is returned mapping by the name. */ - return_format?: - | "markdown" - | "commonmark" - | "raw" - | "text" - | "html2text" - | "bytes"; + return_format?: ReturnFormat | ReturnFormat[]; /** * Specifies whether to only visit the top-level domain. @@ -268,6 +303,10 @@ export interface SpiderParams { */ store_data?: boolean; + /** + * Use webhooks to send data. + */ + webhooks?: WebhookSettings; /** * Configuration settings for GPT (general purpose texture mappings). */ diff --git a/javascript/src/supabase.ts b/javascript/src/supabase.ts deleted file mode 100644 index 89b1df9..0000000 --- a/javascript/src/supabase.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { SupabaseClient } from "@supabase/supabase-js"; - -export class Supabase { - private static instance?: SupabaseClient; - private static initPromise?: Promise; - - // Initialize the Supabase client - static async init(): Promise { - if (Supabase.instance) { - return; - } - - if (Supabase.initPromise) { - return Supabase.initPromise; - } - - Supabase.initPromise = (async () => { - const windowExists = typeof window !== "undefined"; - const { createClient } = await import("@supabase/supabase-js"); - - try { - const response = await fetch("https://api.spider.cloud/data/anon_key"); - - if (!response.ok) { - throw new Error(`Failed to fetch anon key: ${response.statusText}`); - } - - const { data } = await response.json(); - - Supabase.instance = createClient( - "https://api-data.spider.cloud", - String(data), - { - auth: { - persistSession: windowExists, - autoRefreshToken: windowExists, - }, - }, - ); - } catch (error) { - Promise.reject("Failed to initialize Supabase client: " + error); - } finally { - Supabase.initPromise = undefined; // Clear the init promise - } - })(); - - return Supabase.initPromise; - } - - // Get the Supabase client instance - static get client(): SupabaseClient { - if (!Supabase.instance) { - throw new Error( - "Supabase client is not initialized. Call Supabase.init() first.", - ); - } - return Supabase.instance; - } -} diff --git a/python/README.md b/python/README.md index e828a41..2e387b4 100644 --- a/python/README.md +++ b/python/README.md @@ -246,21 +246,6 @@ crawler_params = { links = app.crawl(url, crawler_params, True, "application/jsonl") ``` -## Supabase - -You can use [Supabase](https://supabase.com/docs/reference/python) to directly connect to instances and write your own logic. First, you need to run `pip install supabase` since this package does not include the dependency by default. This keeps the bundle size small and allows for lazy imports of the client. - -```python - spider.init_supabase() - - auth_response = spider.supabase.auth.sign_in_with_password( - { - "email": os.getenv("SPIDER_EMAIL", ""), - "password": os.getenv("SPIDER_PASSWORD", ""), - } - ) -``` - ## Error Handling The SDK handles errors returned by the Spider Cloud API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. diff --git a/python/requirements.txt b/python/requirements.txt index 1cd37db..24668a3 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,6 +1,5 @@ pytest pytest-asyncio python-dotenv -supabase aiohttp python-dotenv \ No newline at end of file diff --git a/python/spider/spider.py b/python/spider/spider.py index 3f2c86c..b4ba6ea 100644 --- a/python/spider/spider.py +++ b/python/spider/spider.py @@ -1,8 +1,6 @@ import os, requests, json, logging from typing import Optional, Dict from spider.spider_types import RequestParamsDict, JsonCallback, QueryRequest -from spider.supabase_client import Supabase - class Spider: def __init__(self, api_key: Optional[str] = None): @@ -16,30 +14,6 @@ def __init__(self, api_key: Optional[str] = None): if self.api_key is None: raise ValueError("No API key provided") - def init_supabase(self): - """ - Initialize the Supabase client if it is not already initialized. - - This method is optional and only needs to be called if you plan to use the Supabase client. - - :raises ImportError: If the 'supabase' package is not installed. - :raises Exception: If there is an issue fetching the anon key or initializing the client. - """ - Supabase.init() - - @property - def supabase(self): - """ - Get the Supabase client instance. - - This property is optional and only needs to be accessed if you plan to use the Supabase client. - Ensure that the Supabase client has been initialized by calling 'init_supabase()' before accessing this property. - - :return: The initialized Supabase client instance. - :raises Exception: If the Supabase client has not been initialized. - """ - return Supabase.get_client() - def api_post( self, endpoint: str, diff --git a/python/spider/spider_types.py b/python/spider/spider_types.py index fb3d3e6..4da5c60 100644 --- a/python/spider/spider_types.py +++ b/python/spider/spider_types.py @@ -111,6 +111,21 @@ class WaitForDict(TypedDict, total=False): page_navigations: Optional[bool] +@dataclass +class WebhookSettings: + # The destination where the webhook data is sent via HTTP POST. + destination: str + # Flag to trigger an action when all credits are depleted + on_credits_depleted: bool + # Flag to trigger when half of the credits are depleted + on_credits_half_depleted: bool + # Flag to notify on website status update events + on_website_status: bool + # Flag to send information (links, bytes) about a new page find + on_find: bool + # Flag to handle the metadata of a found page + on_find_metadata: bool + class CSSSelector(TypedDict): """ Represents a set of CSS selectors grouped under a common name. @@ -125,6 +140,7 @@ class CSSSelector(TypedDict): # - Values are lists of CSSSelector items CSSExtractionMap = Dict[str, List[CSSSelector]] +ReturnFormat = Literal["raw", "markdown", "commonmark", "html2text", "text", "bytes"]; class RequestParamsDict(TypedDict, total=False): # The URL to be crawled. @@ -138,7 +154,7 @@ class RequestParamsDict(TypedDict, total=False): # The format in which the result should be returned. return_format: Optional[ - Literal["raw", "markdown", "commonmark", "html2text", "text", "bytes"] + ReturnFormat | List[ReturnFormat] ] # Specifies whether to only visit the top-level domain. @@ -231,6 +247,9 @@ class RequestParamsDict(TypedDict, total=False): # Returns the OpenAI embeddings for the title and description. Other values, such as keywords, may also be included. Requires the `metadata` parameter to be set to `true`. return_embeddings: Optional[bool] + # Use webhooks to send data to another location via POST. + webhooks: Optional[WebhookSettings] + # Returns the link(s) found on the page that match the crawler query. return_page_links: Optional[bool] diff --git a/python/spider/supabase_client.py b/python/spider/supabase_client.py deleted file mode 100644 index b3875ad..0000000 --- a/python/spider/supabase_client.py +++ /dev/null @@ -1,45 +0,0 @@ -import importlib, requests, os - - -class Supabase: - _client = None - - @staticmethod - def init(): - if Supabase._client is None: - Supabase._initialize_client() - - @staticmethod - def _initialize_client(): - try: - supabase_module = importlib.import_module("supabase") - create_client = getattr(supabase_module, "create_client") - client_options = getattr(supabase_module, "ClientOptions") - except ImportError: - raise ImportError( - "Supabase client is not available. Install it using 'pip install supabase'" - ) - - response = requests.get("https://api.spider.cloud/data/anon_key") - if not response.ok: - raise Exception(f"Failed to fetch anon key: {response.status_code}") - - data = response.json().get("data") - if not data: - raise Exception("Anon key data is missing in the response") - - auto_refresh_token = os.getenv("SUPABASE_AUTO_REFRESH_TOKEN", "True").lower() == "true" - - Supabase._client = create_client( - "https://api-data.spider.cloud", - str(data), - options=client_options(auto_refresh_token=auto_refresh_token), - ) - - @staticmethod - def get_client(): - if Supabase._client is None: - raise Exception( - "Supabase client is not initialized. Call Supabase.init() first." - ) - return Supabase._client diff --git a/python/tests/test_async_spider.py b/python/tests/test_async_spider.py index 785ee94..f5c05f0 100644 --- a/python/tests/test_async_spider.py +++ b/python/tests/test_async_spider.py @@ -270,31 +270,6 @@ async def mock_request(*args, **kwargs): async for response in async_spider.create_signed_url(params={"domain": "example.com"}): assert response == b"mocked raw data" -# @pytest.mark.asyncio -# async def test_supabase_init(async_spider): -# with patch('spider.supabase_client.Supabase._initialize_client') as mock_init: -# await async_spider.init_supabase() -# mock_init.assert_called_once() - -# def test_supabase_get_client_not_initialized(async_spider): -# with pytest.raises(Exception, match="Supabase client is not initialized"): -# _ = async_spider.supabase - -# @pytest.mark.asyncio -# async def test_stream_reader(): -# async_spider = AsyncSpider(api_key="test_api_key") -# mock_response = AsyncMock(spec=aiohttp.ClientResponse) -# mock_response.__aiter__.return_value = [b'{"key": "value"}\n', b'{"key2": "value2"}\n'].__aiter__() - -# callback_data = [] -# def callback(json_obj): -# callback_data.append(json_obj) - -# await async_spider._stream_reader(mock_response, callback) -# assert len(callback_data) == 2 -# assert callback_data[0] == {"key": "value"} -# assert callback_data[1] == {"key2": "value2"} - @pytest.mark.asyncio async def test_handle_error(): async_spider = AsyncSpider(api_key="test_api_key") diff --git a/python/tests/test_spider.py b/python/tests/test_spider.py index ef7aee5..2262a34 100644 --- a/python/tests/test_spider.py +++ b/python/tests/test_spider.py @@ -263,15 +263,6 @@ def test_create_signed_url(mock_get, spider): assert response == b"mocked raw data" mock_get.assert_called_once() -def test_supabase_init(spider): - with patch('spider.supabase_client.Supabase._initialize_client') as mock_init: - spider.init_supabase() - mock_init.assert_called_once() - -def test_supabase_get_client_not_initialized(spider): - with pytest.raises(Exception, match="Supabase client is not initialized"): - spider.supabase - def test_stream_reader(): spider = Spider(api_key="test_api_key") mock_response = MagicMock() diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c7102b3..dab4464 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -25,9 +25,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" @@ -64,15 +64,15 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "cc" -version = "1.1.18" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "shlex", ] @@ -168,30 +168,30 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -200,21 +200,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-io", @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "h2" @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" [[package]] name = "hermit-abi" @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "hyper" @@ -370,9 +370,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -383,7 +383,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -400,9 +399,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", "hashbrown", @@ -410,9 +409,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "itoa" @@ -422,9 +421,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "0cb94a0ffd3f3ee755c20f7d8752f45cac88605a4dcf808abcff72873296ec7b" dependencies = [ "wasm-bindgen", ] @@ -437,9 +436,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "linux-raw-sys" @@ -505,18 +504,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.20.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ea5043e58958ee56f3e15a90aee535795cd7dfd319846288d93c5b57d85cbe" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl" @@ -568,26 +567,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -602,15 +581,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -626,9 +605,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64", "bytes", @@ -705,9 +684,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "once_cell", "rustls-pki-types", @@ -718,19 +697,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -751,9 +729,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -773,9 +751,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -858,7 +836,7 @@ dependencies = [ [[package]] name = "spider-client" -version = "0.1.19" +version = "0.1.20" dependencies = [ "dotenv", "lazy_static", @@ -883,9 +861,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -924,9 +902,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -1022,27 +1000,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -1076,9 +1033,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -1088,9 +1045,9 @@ checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] @@ -1135,9 +1092,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "ef073ced962d62984fb38a36e5fdc1a2b23c9e0e1fa0689bb97afa4202ef6887" dependencies = [ "cfg-if", "once_cell", @@ -1146,9 +1103,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "c4bfab14ef75323f4eb75fa52ee0a3fb59611977fd3240da19b2cf36ff85030e" dependencies = [ "bumpalo", "log", @@ -1161,9 +1118,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "65471f79c1022ffa5291d33520cbbb53b7687b01c2f8e83b57d102eed7ed479d" dependencies = [ "cfg-if", "js-sys", @@ -1173,9 +1130,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "a7bec9830f60924d9ceb3ef99d55c155be8afa76954edffbb5936ff4509474e7" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1183,9 +1140,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "4c74f6e152a76a2ad448e223b0fc0b6b5747649c3d769cc6bf45737bf97d0ed6" dependencies = [ "proc-macro2", "quote", @@ -1196,15 +1153,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "a42f6c679374623f295a8623adfe63d9284091245c3504bde47c17a3ce2777d9" [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" dependencies = [ "futures-util", "js-sys", @@ -1215,9 +1172,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "44188d185b5bdcae1052d08bcbcf9091a5524038d4572cc4f4f2bb9d5554ddd9" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 3601edc..cc64ad5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spider-client" -version = "0.1.19" +version = "0.1.20" edition = "2021" authors = [ "j-mendez "] description = "Spider Cloud client" diff --git a/rust/src/lib.rs b/rust/src/lib.rs index fbb3725..8c26c5f 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -65,6 +65,7 @@ use reqwest::Client; use reqwest::{Error, Response}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use std::default; use tokio_stream::StreamExt; /// Structure representing the Chunking algorithm dictionary. @@ -201,6 +202,39 @@ pub struct CSSSelector { // Define the CSSExtractionMap type pub type CSSExtractionMap = HashMap>; +/// Represents the settings for a webhook configuration +#[derive(Debug, Default, Deserialize, Serialize, Clone)] +pub struct WebhookSettings { + /// The destination where the webhook information will be sent + destination: String, + /// Trigger an action when all credits are depleted + on_credits_depleted: bool, + /// Trigger an action when half of the credits are depleted + on_credits_half_depleted: bool, + /// Trigger an action on a website status update event + on_website_status: bool, + /// Send information about a new page find (such as links and bytes) + on_find: bool, + /// Handle the metadata of a found page + on_find_metadata: bool, +} + +/// Send multiple return formats. +#[derive(Debug, Deserialize, Serialize, Clone)] +#[serde(untagged)] +pub enum ReturnFormatHandling { + /// A single return item. + Single(ReturnFormat), + /// Multiple return formats. + Multi(std::collections::HashSet), +} + +impl Default for ReturnFormatHandling { + fn default() -> ReturnFormatHandling { + ReturnFormatHandling::Single(ReturnFormat::Raw) + } +} + /// Structure representing request parameters. #[derive(Debug, Default, Deserialize, Serialize, Clone)] pub struct RequestParams { @@ -215,7 +249,7 @@ pub struct RequestParams { pub limit: Option, #[serde(default)] /// The format in which the result should be returned. - pub return_format: Option, + pub return_format: Option, #[serde(default)] /// Specifies whether to only visit the top-level domain. pub tld: Option, @@ -253,6 +287,9 @@ pub struct RequestParams { /// Specifies whether anti-bot measures should be used. pub anti_bot: Option, #[serde(default)] + /// Specifies whether to send data via webhooks. + pub webhooks: Option, + #[serde(default)] /// Specifies whether to include metadata in the response. pub metadata: Option, #[serde(default)] @@ -408,7 +445,7 @@ pub enum RequestType { } /// Enum representing different return formats. -#[derive(Default, Debug, Deserialize, Serialize, Clone)] +#[derive(Default, Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Hash)] #[serde(rename_all = "lowercase")] pub enum ReturnFormat { #[default]