Merge branch 'development' of github.com:Oneirocom/Magick into develo…

…pment

apps/embedder/src/server/api/packs/[id]/loaders.post.ts

@@ -8,6 +8,28 @@ import { embedderDb, Loader, Pack } from '@magickml/embedder-db-pg'
 import { eq, and } from 'drizzle-orm'
 import { useBullMQ, createJob } from '@magickml/embedder-queue'
 import { randomUUID } from 'crypto'
+import { Storage } from '@google-cloud/storage'
+
+const storage = new Storage({
+  projectId: process.env.GOOGLE_CLOUD_PROJECT_ID,
+  credentials: {
+    client_email: process.env.GOOGLE_CLOUD_CLIENT_EMAIL,
+    private_key: process.env.GOOGLE_CLOUD_PRIVATE_KEY?.replace(/\\n/g, '\n'),
+  },
+})
+
+async function generateV4ReadSignedUrl(bucketName: string, fileName: string) {
+  // Get a v4 signed URL for reading the file
+  const [url] = await storage
+    .bucket(bucketName)
+    .file(fileName)
+    .getSignedUrl({
+      version: 'v4',
+      action: 'read',
+      expires: Date.now() + 15 * 60 * 1000, // 15 minutes
+    })
+  return url
+}
 
 export default defineEventHandler(async event => {
   // validate
@@ -19,6 +41,33 @@ export default defineEventHandler(async event => {
     })
   }
 
+  let url
+  if (parse.data.isUpload) {
+    if (!parse.data.path) {
+      throw createError({
+        statusCode: 400,
+        message: 'Missing path for upload',
+      })
+    }
+
+    url = await generateV4ReadSignedUrl(
+      process.env.GOOGLE_PRIVATE_BUCKET_NAME || '',
+      parse.data.path
+    )
+
+    // set the loaders filePathOrUrl to the signed URL
+    if ('filePathOrUrl' in parse.data.config) {
+      parse.data.config.filePathOrUrl = url
+    } else {
+      // 'filePathOrUrl' should cover all cases
+      // if we get here its probably a youtube/etc/etc loader
+      throw createError({
+        statusCode: 400,
+        message: 'Invalid loader type with isUpload',
+      })
+    }
+  }
+
   const { entity, owner } = authParse(event.context)
   const id = idParse(event.context.params?.id)
 
@@ -36,6 +85,9 @@ export default defineEventHandler(async event => {
       message: 'Pack not found',
     })
   }
+  // remove isUpload and path from the loader config
+  delete parse.data.isUpload
+  delete parse.data.path
 
   // Serialize loader into DB
   const loader = await embedderDb

package.json

@@ -131,6 +131,9 @@
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/modifiers": "^7.0.0",
     "@dnd-kit/sortable": "^8.0.0",
+    "@dnd-kit/utilities": "^3.2.2",
+    "@docusaurus/core": "3.0.1",
+    "@docusaurus/preset-classic": "3.0.1",
     "@emotion/styled": "11.11.0",
     "@esbuild-plugins/node-globals-polyfill": "^0.2.3",
     "@esbuild-plugins/node-modules-polyfill": "^0.2.2",

packages/client/windows/knowledge/src/lib/_components/fileDropper.tsx

@@ -1,8 +1,10 @@
-import React, { useCallback } from 'react'
+import React, { useCallback, useState } from 'react'
 import { useDropzone, Accept } from 'react-dropzone'
 
 type FileDropperProps = {
-  handleFileUpload: (files: File[], type?: string) => void
+  handleFileUpload: (
+    event: React.ChangeEvent<HTMLInputElement>
+  ) => Promise<void>
   accept?: Accept
   maxSize?: number
   maxFiles?: number
@@ -45,11 +47,29 @@ const FileDropper: React.FC<FileDropperProps> = ({
   ],
   type,
 }) => {
+  const [uploadStatus, setUploadStatus] = useState<
+    'idle' | 'loading' | 'loaded'
+  >('idle')
+
   const onDrop = useCallback(
-    (acceptedFiles: File[]) => {
-      handleFileUpload(acceptedFiles, type)
+    async (acceptedFiles: File[]) => {
+      setUploadStatus('loading')
+      // Create a synthetic event
+      const syntheticEvent = {
+        target: {
+          files: acceptedFiles,
+        },
+      } as unknown as React.ChangeEvent<HTMLInputElement>
+
+      try {
+        await handleFileUpload(syntheticEvent)
+        setUploadStatus('loaded')
+      } catch (error) {
+        console.error('File upload failed:', error)
+        setUploadStatus('idle')
+      }
     },
-    [handleFileUpload, type]
+    [handleFileUpload]
   )
 
   const { getRootProps, getInputProps, isDragActive } = useDropzone({
@@ -59,6 +79,17 @@ const FileDropper: React.FC<FileDropperProps> = ({
     maxFiles,
   })
 
+  const getUploadStatusText = () => {
+    switch (uploadStatus) {
+      case 'loading':
+        return 'Uploading...'
+      case 'loaded':
+        return 'File uploaded successfully!'
+      default:
+        return isDragActive ? 'Release to drop the files here' : dropzoneText
+    }
+  }
+
   return (
     <div
       className={`flex flex-col items-center justify-center w-full ${className}`}
@@ -69,37 +100,58 @@ const FileDropper: React.FC<FileDropperProps> = ({
       </p>
       <div
         {...getRootProps()}
-        className={`flex flex-col items-center justify-center w-full h-64 bg-white border-2 rounded-lg cursor-pointer border-secondary-da hover:border-secondary-highlight dark:bg-card-main text-center px-8 ${className}`}
+        className={`flex flex-col items-center justify-center w-full h-64 bg-white border-2 rounded-lg cursor-pointer border-secondary-da hover:border-secondary-highlight dark:bg-card-main text-center px-8 ${className} ${
+          uploadStatus === 'loading' ? 'opacity-50' : ''
+        }`}
       >
-        <input {...getInputProps()} />
+        <input {...getInputProps()} disabled={uploadStatus === 'loading'} />
         <div className="flex flex-col items-center justify-center pt-5 pb-6">
-          <svg
-            aria-hidden="true"
-            className="w-10 h-10 mb-3 text-secondary-highlight"
-            fill="none"
-            stroke="currentColor"
-            viewBox="0 0 24 24"
-            xmlns="http://www.w3.org/2000/svg"
-          >
-            <path
-              strokeLinecap="round"
-              strokeLinejoin="round"
-              strokeWidth="2"
-              d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12"
-            ></path>
-          </svg>
+          {uploadStatus === 'loading' ? (
+            <svg
+              className="animate-spin h-10 w-10 text-secondary-highlight"
+              xmlns="http://www.w3.org/2000/svg"
+              fill="none"
+              viewBox="0 0 24 24"
+            >
+              <circle
+                className="opacity-25"
+                cx="12"
+                cy="12"
+                r="10"
+                stroke="currentColor"
+                strokeWidth="4"
+              ></circle>
+              <path
+                className="opacity-75"
+                fill="currentColor"
+                d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
+              ></path>
+            </svg>
+          ) : (
+            <svg
+              aria-hidden="true"
+              className="w-10 h-10 mb-3 text-secondary-highlight"
+              fill="none"
+              stroke="currentColor"
+              viewBox="0 0 24 24"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                strokeLinecap="round"
+                strokeLinejoin="round"
+                strokeWidth="2"
+                d="M7 16a4 4 0 01-.88-7.903A5 5 0 1115.9 6L16 6a5 5 0 011 9.9M15 13l-3-3m0 0l-3 3m3-3v12"
+              ></path>
+            </svg>
+          )}
           <p className="mb-2 text-sm text-black dark:text-white">
-            {isDragActive ? (
-              <span>Release to drop the files here</span>
-            ) : (
-              <>
-                <span className="font-semibold">{dropzoneText}</span>
-              </>
-            )}
-          </p>
-          <p className="text-xs text-secondary-highlight">
-            {fileTypes.join(', ')}
+            {getUploadStatusText()}
           </p>
+          {uploadStatus === 'idle' && (
+            <p className="text-xs text-secondary-highlight">
+              {type ? type : fileTypes.join(', ')}
+            </p>
+          )}
         </div>
       </div>
     </div>