importer

api/src/lib/files/S3Path.test.ts

@@ -0,0 +1,25 @@
+import S3Path from 'src/lib/files/S3Path'
+
+describe('paths', () => {
+  it('getBasePath', () => {
+    expect(S3Path.getBasePath('a/a.jpg')).toEqual('a')
+    expect(S3Path.getBasePath('coucou/caca/a.jpg')).toEqual('coucou/caca')
+    expect(S3Path.getBasePath('a/b/c/d/e/a.jpg')).toEqual('a/b/c/d/e')
+    expect(S3Path.getBasePath('a.jpg')).toEqual('')
+    expect(S3Path.getBasePath('a//a.jpg')).toEqual('a')
+    expect(S3Path.getBasePath('/a/b/he.jpg')).toEqual('a/b')
+    expect(S3Path.getBasePath('/a/b/he')).toEqual('a/b')
+  })
+  it('getFileName', () => {
+    expect(S3Path.getFileName('a/a.jpg')).toEqual('a.jpg')
+    expect(S3Path.getFileName('coucou/caca/a.jpg')).toEqual('a.jpg')
+    expect(S3Path.getFileName('coucou/caca/df/fds/ds/a.jpg')).toEqual('a.jpg')
+    expect(S3Path.getFileName('a//a.jpg')).toEqual('a.jpg')
+  })
+  it('getPath', () => {
+    expect(S3Path.getPath('a', 'b.zip')).toEqual('a/b.zip')
+    expect(S3Path.getPath('a/b', 'b.zip')).toEqual('a/b/b.zip')
+    expect(S3Path.getPath('a/b/', 'b.zip')).toEqual('a/b/b.zip')
+    expect(S3Path.getPath('/', 'b.zip')).toEqual('b.zip')
+  })
+})

api/src/lib/files/S3Path.ts

@@ -0,0 +1,32 @@
+export default {
+  getBasePath(path: string) {
+    const basePath = path
+      .split('/')
+      .filter((p) => p !== '')
+      .slice(0, -1)
+      .join('/')
+
+    return basePath
+  },
+
+  getFileName(path: string) {
+    const fileName = path.split('/').slice(-1)[0]
+
+    return fileName
+  },
+
+  getPath(basePath: string | undefined | null, fileName: string) {
+    let cleanBasePath = basePath || ''
+    if (cleanBasePath.length > 0 && cleanBasePath.slice(-1) === '/') {
+      cleanBasePath = cleanBasePath.slice(0, -1)
+    }
+
+    let path = ''
+    if (cleanBasePath.length > 0) {
+      path = cleanBasePath + '/'
+    }
+    path = path + fileName
+
+    return path
+  },
+}

api/src/lib/files/s3.ts

@@ -76,8 +76,16 @@ export class S3Lib {
       Bucket: this.bucket,
       Key,
     }
-    const res = await promisify(this.client, 'headObject')(params)
-    return res
+    try {
+      const res = await promisify(this.client, 'headObject')(params)
+      return res
+    } catch (error) {
+      if (error.code === 'NotFound') {
+        return null
+      } else {
+        throw error
+      }
+    }
   }
 
   async delete(Key: string): Promise<void> {

api/src/lib/importer/importFile.ts

@@ -0,0 +1,79 @@
+import fpath from 'path'
+import { open } from 'src/lib/files/fs'
+import ft from 'file-type'
+import S3Path from 'src/lib/files/S3Path'
+import { db } from 'src/lib/db'
+import {
+  isFileTypeExcluded,
+  isPathExcluded,
+} from 'src/lib/importer/supportedFiles'
+import { uploadImage } from 'src/lib/importer/uploadImages'
+import { getMetadata } from 'src/lib/images/metadata'
+import { createImageTags } from 'src/lib/importer/tagger'
+
+export type Task = {
+  path: string
+}
+
+export enum TaskResult {
+  EXISTING = 'EXISTING',
+  EXCLUDED = 'EXCLUDED',
+  NO_DATE = 'NO_DATE',
+  UPLOADED = 'UPLOADED',
+}
+
+export const getImportWorker = ({ logger, prefix, rootDir }) =>
+  async function importFile({ path }: Task) {
+    let fd
+    try {
+      const fullPath = fpath.resolve(rootDir, path)
+      logger.debug(`Uploading image ${fullPath} ...`)
+
+      if (isPathExcluded(fullPath)) return TaskResult.EXCLUDED
+
+      const s3path = S3Path.getPath(prefix, path)
+
+      const imageExisting = await db.image.findUnique({
+        where: {
+          path: s3path,
+        },
+      })
+      if (imageExisting) return TaskResult.EXISTING
+
+      fd = await open(fullPath, 'r')
+
+      const buffer = await fd.readFile()
+      const fileType = await ft.fromBuffer(buffer)
+
+      if (isFileTypeExcluded(fileType.ext)) return TaskResult.EXCLUDED
+
+      const imageMetadata = await getMetadata(buffer)
+      if (!imageMetadata.parsed.date) {
+        return TaskResult.NO_DATE
+      }
+
+      const stat = await fd.stat()
+      const metadata = {
+        created_at: stat.birthtime.toISOString(),
+        modified_at: stat.mtime.toISOString(),
+      }
+
+      await uploadImage(s3path, buffer, metadata, fileType.mime)
+
+      const image = await db.image.create({
+        data: {
+          path: s3path,
+          dateTaken: imageMetadata.parsed.date.capture,
+          metadata: imageMetadata.raw,
+        },
+      })
+
+      await createImageTags(image, imageMetadata)
+
+      logger.debug(`uploaded image ${fullPath}`)
+
+      return TaskResult.UPLOADED
+    } finally {
+      await fd?.close()
+    }
+  }

api/src/lib/importer/importer.ts

@@ -0,0 +1,50 @@
+import { parallel } from 'src/lib/async'
+import { logger as parentLogger } from 'src/lib/logger'
+import { getImportWorker, Task, TaskResult } from 'src/lib/importer/importFile'
+import { listTasks } from 'src/lib/importer/listTasks'
+
+const logger = parentLogger.child({ module: 'UPLOADER' })
+
+const PARALLEL_LIMIT = 5
+
+export async function importer({
+  rootDir,
+  prefix,
+}: {
+  rootDir: string
+  prefix?: string
+}) {
+  logger.info('Uploader script started')
+
+  logger.debug('Getting file list from file system...')
+  const tasks = await listTasks(rootDir)
+  logger.debug({ filesLength: tasks.length }, 'uploading files...')
+
+  const uploadFileWorker = getImportWorker({
+    logger,
+    prefix,
+    rootDir,
+  })
+
+  const parallelActions = await parallel<Task, TaskResult>(
+    tasks,
+    PARALLEL_LIMIT,
+    uploadFileWorker,
+    true
+  )
+
+  const uploadResult = await parallelActions.finished()
+  if (uploadResult.errors.length)
+    logger.error({ errors: uploadResult.errors }, 'errors while uploading')
+
+  const uploaded = uploadResult.successes.filter(
+    (s) => s.result === TaskResult.UPLOADED
+  ).length
+  const existing = uploadResult.successes.filter(
+    (s) => s.result === TaskResult.EXISTING
+  ).length
+
+  logger.info(
+    `Upload finished: ${uploaded} uploaded, ${existing} existing, ${uploadResult.errors.length} errors`
+  )
+}

api/src/lib/importer/listTasks.ts

@@ -0,0 +1,8 @@
+import { listDirRecursive } from 'src/lib/files/fs'
+import { Task } from 'src/lib/importer/uploadFile'
+
+export async function listTasks(rootDir: string) {
+  const files = await listDirRecursive(rootDir)
+  const tasks: Task[] = files.map((path) => ({ path }))
+  return tasks
+}

api/src/lib/importer/supportedFiles.ts

@@ -0,0 +1,18 @@
+import fpath from 'path'
+
+export const ACCEPTED_EXTENSIONS = ['jpg', 'jpg', 'png', 'webp', 'tif']
+export const EXCLUDED_FILES = ['.DS_Store']
+
+export function isPathExcluded(path: string) {
+  const parsed = fpath.parse(path)
+
+  if (EXCLUDED_FILES.includes(parsed.base)) return true
+
+  return false
+}
+
+export function isFileTypeExcluded(ext: string) {
+  if (!ACCEPTED_EXTENSIONS.includes(ext)) return true
+
+  return false
+}

api/src/lib/importer/uploadImages.ts

@@ -0,0 +1,8 @@
+import { Buckets } from 'src/lib/files/s3'
+import { getMiniature } from 'src/lib/images/miniature'
+
+export async function uploadImage(path, imageBuffer: Buffer, metadata, mime) {
+  await Buckets.photos.put(path, imageBuffer, metadata, mime)
+  const miniature = await getMiniature(imageBuffer)
+  await Buckets.miniatures.put(path, miniature.buffer, null, miniature.mime)
+}