scan from s3

.env.defaults

@@ -31,7 +31,7 @@ WEBHOOK_SECRET=THIS_IS_NOT_SECRET_PLEASE_CHANGE
 
 FILESYSTEM_FOLDER=./fs
 # STATIC_SERVER_URL=http://127.0.0.1:8080
-STATIC_SERVER_URL=http://localhost:9000/photos/fs
+STATIC_SERVER_URL=http://localhost:9000/photos
 
 S3_URL=http://localhost:9000
 S3_ACCESS_KEY=minio_client

api/src/lib/files/index.ts

@@ -30,6 +30,15 @@ type RecursiveFileIterator = {
   next: () => Promise<string | null>
 }
 
+/*
+const a = await listDirRecursiveIter(path)
+
+let b = await a.next()
+while (b) {
+  console.log(b)
+  b = await a.next()
+}
+*/
 export async function listDirRecursiveIter(
   rootPath: string
 ): Promise<RecursiveFileIterator> {

api/src/lib/images/metadata.ts

@@ -2,16 +2,18 @@ import moment from 'moment'
 import { ExifImage } from 'exif'
 
 type ImageMetadata = {
-  image: object
-  exif: object
-  gps: object
+  image: any
+  exif: any
+  gps: any
 }
 
 type ParsedImageMetadata = {
   CreateDate?: Date
 }
 
-export async function getMetadata(path: string): Promise<ImageMetadata> {
+export async function getMetadata(
+  path: Buffer | string
+): Promise<ImageMetadata> {
   return new Promise((resolve, reject) => {
     try {
       // eslint-disable-next-line no-new

api/src/lib/images/scanner.ts

@@ -1,33 +1,24 @@
 import type { Prisma } from '@prisma/client'
 
-import { listDirRecursive } from 'src/lib/files'
 import { db } from 'src/lib/db'
 
 import { getMetadata, parseMetadata } from 'src/lib/images/metadata'
-
-/*
-const a = await listDirRecursiveIter(path)
-
-let b = await a.next()
-while (b) {
-  console.log(b)
-  b = await a.next()
-}
-*/
+import { S3Lib } from 'src/lib/s3'
 
 export async function scanFiles() {
-  const path = process.env['FILESYSTEM_FOLDER']
+  console.log('Scanner script started')
 
-  const files = await listDirRecursive(path)
+  const files = await S3Lib.list()
 
   console.log('emptying db')
   await db.image.deleteMany({})
 
-  console.log('importing files', files)
+  console.log('importing files from s3', files.length)
 
   await Promise.all(
     files.map(async (imagePath: Prisma.ImageCreateInput['path']) => {
-      const metadata = await getMetadata(`${path}/${imagePath}`)
+      const imageBuffer = await S3Lib.get(imagePath)
+      const metadata = await getMetadata(imageBuffer)
       const parsedMetadata = parseMetadata(metadata)
 
       const record = await db.image.create({
@@ -38,7 +29,7 @@ export async function scanFiles() {
           metadataJson: JSON.stringify(metadata) || '',
         },
       })
-      console.log(record.path)
+      console.log('added image', record.path)
     })
   )
 

api/src/lib/s3/index.ts

@@ -16,14 +16,62 @@ const promiseSimple = (resolve, reject) => (err, res) => {
     resolve(res)
   }
 }
-export async function listS3(prefix) {
-  return new Promise((resolve, reject) => {
-    const bucketParams = {
+
+const promisify =
+  (fn) =>
+  (params): ReturnType<typeof fn> => {
+    return new Promise((resolve, reject) => {
+      fn.bind(s3)(params, (err, res) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(res)
+        }
+      })
+    })
+  }
+
+export const S3Lib = {
+  async list(Prefix?: string): Promise<string[]> {
+    const params = {
       Bucket: process.env['S3_BUCKET_NAME'],
-      Prefix: prefix,
+      Prefix,
     }
-    s3.listObjects(bucketParams, promiseSimple(resolve, reject))
-  })
+    const res = await promisify(s3.listObjects)(params)
+    return res.Contents.map((c) => c.Key)
+  },
+
+  async get(Key: string): Promise<Buffer> {
+    const params = {
+      Bucket: process.env['S3_BUCKET_NAME'],
+      Key,
+    }
+    const res = await promisify(s3.getObject)(params)
+    return res.Body
+  },
+  async delete(Key: string): Promise<void> {
+    const params = {
+      Bucket: process.env['S3_BUCKET_NAME'],
+      Key,
+    }
+    await promisify(s3.deleteObject)(params)
+  },
+  async put(Key: string, Object: string): Promise<void> {
+    const params = {
+      Bucket: process.env['S3_BUCKET_NAME'],
+      Key,
+      Object,
+    }
+    await promisify(s3.putObject)(params)
+  },
+  async update(Key: string, Object: string): Promise<void> {
+    const params = {
+      Bucket: process.env['S3_BUCKET_NAME'],
+      Key,
+      Object,
+    }
+    await promisify(s3.putObject)(params)
+  },
 }
 
 /*

package.json

@@ -26,8 +26,8 @@
     "start": "yarn rw dev",
     "dockers": "docker-compose up -d",
     "cleandb": "docker-compose down -v && rm -rf ./api/db/migrations && yarn rw migrate dev --name init",
-    "indexer": "yarn rw exec FileIndexer",
-    "static": "yarn rw exec FileServer",
+    "scan": "yarn rw exec scan",
+    "static": "yarn rw exec static-server",
     "studio": "yarn rw prisma studio"
   }
 }

scripts/s3.ts

@@ -1,6 +1,8 @@
-import { listS3 } from 'api/src/lib/s3'
+import { S3Lib } from 'api/src/lib/s3'
 
 export default async () => {
-  const res1 = await listS3('fs/a')
+  const res1 = await S3Lib.list('fs/a')
   console.log(res1)
+  const res2 = await S3Lib.get('fs/a/a.JPG')
+  console.log(res2)
 }