create miniature when scan

.env.defaults

@@ -30,11 +30,11 @@ WEBHOOK_SECRET=THIS_IS_NOT_SECRET_PLEASE_CHANGE
 #######
 
 FILESYSTEM_FOLDER=./fs
-# STATIC_SERVER_URL=http://127.0.0.1:8080
-# STATIC_SERVER_URL=http://localhost:9000/photos
-STATIC_SERVER_URL=/s3/photos
+PHOTOS_URL=/s3/photos
+MINIATURES_URL=/s3/miniatures
 
 S3_URL=http://localhost:9000
 S3_ACCESS_KEY=minio_client
 S3_SECRET_KEY=minio_secret
-S3_BUCKET_NAME=photos
+S3_BUCKET_PHOTOS=photos
+S3_BUCKET_MINIATURES=miniatures

api/src/lib/files/s3.ts

@@ -1,20 +1,11 @@
 import { S3 } from 'aws-sdk'
 import { ReadStream } from 'fs'
 
-const s3 = new S3({
-  region: 'local',
-  endpoint: process.env['S3_URL'],
-  accessKeyId: process.env['S3_ACCESS_KEY'],
-  secretAccessKey: process.env['S3_SECRET_KEY'],
-  s3ForcePathStyle: true, // needed with minio?
-  signatureVersion: 'v4',
-})
-
 const promisify =
-  (fn) =>
-  (params): ReturnType<typeof fn> => {
+  (lib, fnName) =>
+  (params): ReturnType<typeof lib['fnName']> => {
     return new Promise((resolve, reject) => {
-      fn.bind(s3)(params, (err, res) => {
+      lib[fnName].bind(lib)(params, (err, res) => {
         if (err) {
           reject(err)
         } else {
@@ -24,74 +15,85 @@ const promisify =
     })
   }
 
-export const S3Lib = {
+export class S3Lib {
+  bucket: string
+  client: S3
+
+  constructor(bucket: string) {
+    this.client = new S3({
+      region: 'local',
+      endpoint: process.env['S3_URL'],
+      accessKeyId: process.env['S3_ACCESS_KEY'],
+      secretAccessKey: process.env['S3_SECRET_KEY'],
+      s3ForcePathStyle: true, // needed with minio?
+      signatureVersion: 'v4',
+    })
+    this.bucket = bucket
+  }
+
   async list(Prefix?: string): Promise<string[]> {
     const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
+      Bucket: this.bucket,
       Prefix,
     }
-    const res = await promisify(s3.listObjects)(params)
+    const res = await promisify(this.client, 'listObjects')(params)
     return res.Contents.map((c) => c.Key)
-  },
+  }
 
   async get(Key: string, Range?: string): Promise<Buffer> {
     const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
+      Bucket: this.bucket,
       Key,
       Range,
     }
-    const res = await promisify(s3.getObject)(params)
+    const res = await promisify(this.client, 'getObject')(params)
     return res.Body
-  },
+  }
+
   async head(Key: string): Promise<S3.Types.HeadObjectOutput> {
     const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
+      Bucket: this.bucket,
       Key,
     }
-    const res = await promisify(s3.headObject)(params)
+    const res = await promisify(this.client, 'headObject')(params)
     return res
-  },
+  }
+
   async delete(Key: string): Promise<void> {
     const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
+      Bucket: this.bucket,
       Key,
     }
-    await promisify(s3.deleteObject)(params)
-  },
+    await promisify(this.client, 'deleteObject')(params)
+  }
+
   async deletePrefix(Prefix: string): Promise<void> {
-    const list = await S3Lib.list(Prefix)
+    const list = await this.list(Prefix)
     if (list.length === 0) return
     const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
+      Bucket: this.bucket,
       Delete: {
         Objects: list.map((i) => ({ Key: i })),
       },
     }
-    await promisify(s3.deleteObjects)(params)
-  },
+    await promisify(this.client, 'deleteObjects')(params)
+  }
+
   async put(
     Key: string,
     Body: string | Buffer | ReadStream,
     Metadata?: Record<string, string>,
     ContentType?: string
   ): Promise<void> {
     const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
+      Bucket: this.bucket,
       Key,
       Body,
       Metadata,
       ContentType,
     }
-    await promisify(s3.putObject)(params)
-  },
-  async update(Key: string, Object: string): Promise<void> {
-    const params = {
-      Bucket: process.env['S3_BUCKET_NAME'],
-      Key,
-      Object,
-    }
-    await promisify(s3.putObject)(params)
-  },
+    await promisify(this.client, 'putObject')(params)
+  }
 }
 
 /*
@@ -108,19 +110,19 @@ export const minioClient = new Client({
 export function listS3(prefix) {
   return new Promise((resolve, reject) => {
     const stream = minioClient.extensions.listObjectsV2WithMetadata(
-      process.env['S3_BUCKET_NAME'],
+      this.bucket,
       prefix,
       true
     )
     stream.on('data', resolve)
     stream.on('error', reject)
   })
 }
-import { S3Client, ListObjectsCommand } from '@aws-sdk/client-s3' // ES Modules import
+import { S3Client, ListObjectsCommand } from '@aws-sdk/client-this.client' // ES Modules import
 import type {
   S3ClientConfig,
   ListObjectsCommandInput,
-} from '@aws-sdk/client-s3'
+} from '@aws-sdk/client-this.client'
 
 const config: S3ClientConfig = {
   region: 'local',
@@ -134,7 +136,7 @@ const client = new S3Client(config)
 
 export async function listS3(prefix) {
   const input: ListObjectsCommandInput = {
-    Bucket: process.env['S3_BUCKET_NAME'],
+    Bucket: this.bucket,
     Prefix: prefix,
   }
   const command = new ListObjectsCommand(input)

api/src/lib/images/constants.ts

@@ -1 +1,3 @@
 export const ACCEPTED_EXTENSIONS = ['jpg', 'png', 'webp', 'tif']
+export const MINIATURE_HEIGHT = 400
+export const MINIATURE_QUALITY = 80

api/src/lib/images/miniature.ts

@@ -0,0 +1,22 @@
+import sharp from 'sharp'
+import { MINIATURE_HEIGHT, MINIATURE_QUALITY } from 'src/lib/images/constants'
+
+type Miniature = {
+  buffer: Buffer
+  mime: string
+}
+export async function getMiniature(imageBuffer: Buffer): Promise<Miniature> {
+  const sharpThumbnail = await sharp(imageBuffer)
+    .resize({
+      height: MINIATURE_HEIGHT,
+    })
+    .webp({
+      quality: MINIATURE_QUALITY,
+    })
+    .toBuffer()
+
+  return {
+    buffer: sharpThumbnail,
+    mime: 'image/webp',
+  }
+}

api/src/lib/images/scanner.ts

@@ -7,9 +7,12 @@ import { getMetadata, ImageMetadata, joinString } from 'src/lib/images/metadata'
 import { S3Lib } from 'src/lib/files/s3'
 import { parallel } from 'src/lib/async'
 import { ACCEPTED_EXTENSIONS } from 'src/lib/images/constants'
+import { getMiniature } from 'src/lib/images/miniature'
 
 const PARALLEL_SCANS = 5
-const BYTES_RANGE = 50000
+
+const s3photos = new S3Lib(process.env['S3_BUCKET_PHOTOS'])
+const s3miniatures = new S3Lib(process.env['S3_BUCKET_MINIATURES'])
 
 async function createImageTags(
   image: Image,
@@ -137,14 +140,19 @@ function getFileInceptionDate(head: Record<string, any>) {
   return inceptionDate
 }
 
+async function createMiniature(imageBuffer: Buffer, path: string) {
+  const miniature = await getMiniature(imageBuffer)
+  await s3miniatures.put(path, miniature.buffer, null, miniature.mime)
+}
+
 async function scanImage(imagePath: Prisma.ImageCreateInput['path']) {
   console.log('- scanning image', imagePath)
 
-  const head = await S3Lib.head(imagePath)
+  const head = await s3photos.head(imagePath)
   if (head.ContentLength === 0) {
     throw new Error('Zero byte file')
   }
-  const imageBuffer = await S3Lib.get(imagePath, `bytes=0-${BYTES_RANGE}`)
+  const imageBuffer = await s3photos.get(imagePath)
 
   const fileType = await ft.fromBuffer(imageBuffer)
   if (!fileType || ACCEPTED_EXTENSIONS.indexOf(fileType.ext) === -1) {
@@ -165,6 +173,7 @@ async function scanImage(imagePath: Prisma.ImageCreateInput['path']) {
   })
 
   await createImageTags(image, imageMetadata, inceptionDate)
+  await createMiniature(imageBuffer, imagePath)
 
   console.log('added image', image.path)
   return imagePath
@@ -178,14 +187,14 @@ export async function scanFiles() {
   await db.tagGroup.deleteMany({})
 
   console.log('Getting file list from S3...')
-  const files = await S3Lib.list('test_upload')
+  const files = await s3photos.list('test_upload')
   console.log('importing files from s3', files.length)
 
   const scanResult = await parallel(files, PARALLEL_SCANS, scanImage)
 
-  console.log('Finished script')
+  if (scanResult.errors.length) console.log('errors:', scanResult.errors)
   console.log(
     `${scanResult.successes.length} success, ${scanResult.errors.length} errors`
   )
-  if (scanResult.errors.length) console.log('errors:', scanResult.errors)
+  console.log('Finished script')
 }

api/src/lib/uploader/upload.ts

@@ -9,6 +9,8 @@ const PARALLEL_UPLOAD = 5
 
 const dir = process.env['FILESYSTEM_FOLDER']
 
+const s3photos = new S3Lib(process.env['S3_BUCKET_PHOTOS'])
+
 async function uploadFile(path: string) {
   let fd
   try {
@@ -30,7 +32,7 @@ async function uploadFile(path: string) {
       modified_at: stat.mtime.toISOString(),
     }
 
-    await S3Lib.put(`test_upload/${path}`, buffer, metadata, fileType.mime)
+    await s3photos.put(`test_upload/${path}`, buffer, metadata, fileType.mime)
 
     return path
   } finally {
@@ -42,7 +44,7 @@ export async function upload() {
   console.log('Uploader script started')
 
   console.log('Emptying bucket...')
-  await S3Lib.deletePrefix('test_upload')
+  await s3photos.deletePrefix('test_upload')
 
   console.log('Getting file list from file system...')
   const files = await listDirRecursive(dir)

docker-compose.yml

@@ -41,8 +41,11 @@ services:
     entrypoint: >
       /bin/sh -c "
       /usr/bin/mc alias set myminio http://minio:9000 minio minio123;
+      /usr/bin/mc admin user add myminio minio_client minio_secret;
       /usr/bin/mc mb myminio/photos;
+      /usr/bin/mc mb myminio/miniatures;
       /usr/bin/mc policy set public myminio/photos;
+      /usr/bin/mc policy set public myminio/miniatures;
       exit 0;
       "
 

redwood.toml

@@ -1,15 +1,8 @@
-# This file contains the configuration settings for your Redwood app.
-# This file is also what makes your Redwood app a Redwood app.
-# If you remove it and try to run `yarn rw dev`, you'll get an error.
-#
-# For the full list of options, see the "App Configuration: redwood.toml" doc:
-# https://redwoodjs.com/docs/app-configuration-redwood-toml
-
 [web]
   title = "Barracuda Photos"
   port = 8910
-  apiUrl = "/.redwood/functions" # you can customise graphql and dbauth urls individually too: see https://redwoodjs.com/docs/app-configuration-redwood-toml#api-paths
-  includeEnvironmentVariables = ['STATIC_SERVER_URL', 'GMAPS_API_KEY'] # any ENV vars that should be available to the web side, see https://redwoodjs.com/docs/environment-variables#web
+  apiUrl = "/.redwood/functions"
+  includeEnvironmentVariables = ['PHOTOS_URL', 'MINIATURES_URL', 'GMAPS_API_KEY']
 [api]
   port = 8911
 [browser]

web/package.json

@@ -28,6 +28,7 @@
     "react": "17.0.2",
     "react-dom": "17.0.2",
     "react-hook-form": "^7.26.1",
-    "react-icons": "^4.3.1"
+    "react-icons": "^4.3.1",
+    "sharp": "^0.30.1"
   }
 }

web/src/components/Image/Images/ImagesItem.tsx

@@ -1,5 +1,5 @@
 import { Link, routes } from '@redwoodjs/router'
-import { getImageUrl } from 'src/lib/static'
+import { getMiniatureUrl } from 'src/lib/static'
 import { Box, Center, Icon, Image } from '@chakra-ui/react'
 
 import { FindImages } from 'types/graphql'
@@ -21,7 +21,7 @@ export const ImagesItem = ({ image }: ImagesItemProps) => {
   } = useSelectContext()
 
   const imageComponent = (
-    <Image src={getImageUrl(image)} alt={image.path} h={250} />
+    <Image src={getMiniatureUrl(image)} alt={image.path} h={250} />
   )
   const imageSelected = useMemo(
     () => isImageSelected(image),

web/src/lib/static.ts

@@ -1,8 +1,14 @@
 import type { FindImageById } from 'types/graphql'
 import type { CellSuccessProps } from '@redwoodjs/web'
 
-const static_server_url = process.env['STATIC_SERVER_URL']
+const PHOTOS_PREFIX = process.env['PHOTOS_URL']
+const MINATURES_PREFIX = process.env['MINIATURES_URL']
 
 export function getImageUrl(image: CellSuccessProps<FindImageById>['image']) {
-  return `${static_server_url}/${image.path}`
+  return `${PHOTOS_PREFIX}/${image.path}`
+}
+export function getMiniatureUrl(
+  image: CellSuccessProps<FindImageById>['image']
+) {
+  return `${MINATURES_PREFIX}/${image.path}`
 }