Skip to content

Commit

Permalink
Merge pull request #27 from hv0905/add-test
Browse files Browse the repository at this point in the history 
Test delete and local get in integrate test and improve readme
  • Loading branch information
@hv0905
hv0905 authored May 14, 2024
2 parents 63b27f3 + ef551cc commit efd3d75
Show file tree
Hide file tree
Showing 13 changed files with 105 additions and 75 deletions.
12 changes: 8 additions & 4 deletions app/Models/api_models/admin_query_params.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,18 @@
class UploadImageModel:
def __init__(self,
url: Optional[str] = Query(None,
description="The image's url. If the image is local, this field will be ignored. Otherwise it is required."),
description="The image's url. If the image is local, this field will be "
"ignored. Otherwise it is required."),
thumbnail_url: Optional[str] = Query(None,
description="The image's thumbnail url. If the image is local, this field will be ignored."),
description="The image's thumbnail url. If the image is local, "
"this field will be ignored."),
categories: Optional[str] = Query(None,
description="The categories of the image. The entries should be seperated by comma."),
description="The categories of the image. The entries should be "
"seperated by comma."),
starred: bool = Query(False, description="If the image is starred."),
local: bool = Query(False,
description="When set to true, the image will be uploaded to local storage. Otherwise, it will only be indexed in the database."),
description="When set to true, the image will be uploaded to local storage. "
"Otherwise, it will only be indexed in the database."),
skip_ocr: bool = Query(False, description="Whether to skip the OCR process.")):
self.url = url
self.thumbnail_url = thumbnail_url
Expand Down
4 changes: 2 additions & 2 deletions app/Models/img_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ def payload(self):
return result

@classmethod
def from_payload(cls, id: str, payload: dict,
def from_payload(cls, img_id: str, payload: dict,
image_vector: Optional[ndarray] = None, text_contain_vector: Optional[ndarray] = None):
# Convert the datetime string back to datetime object
index_date = datetime.fromisoformat(payload['index_date'])
del payload['index_date']
return cls(id=UUID(id),
return cls(id=UUID(img_id),
index_date=index_date,
**payload,
image_vector=image_vector if image_vector is not None else None,
Expand Down
3 changes: 2 additions & 1 deletion app/Services/ocr_services.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ def __init__(self):
needWarmUp=True,
devices=self._device,
warmup_size=(960, 960),
model_local_dir=config.model.easypaddleocr if config.model.easypaddleocr else None)
model_local_dir=config.model.easypaddleocr if
config.model.easypaddleocr else None)
logger.success("EasyPaddleOCR loaded successfully")

@staticmethod
Expand Down
2 changes: 1 addition & 1 deletion app/Services/storage/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,6 @@ def __init__(self):
case StorageMode.DISABLED:
return
case _:
raise NotImplementedError(f"Storage method {config.storage.method} not implemented. "
raise NotImplementedError(f"Storage method {config.storage.method} not implemented. "
f"Available methods: local, s3")
self.active_storage.pre_check()
2 changes: 0 additions & 2 deletions app/Services/storage/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@
import os
from typing import TypeVar, Generic, TypeAlias, Optional, AsyncGenerator

from app.Models.img_data import ImageData

FileMetaDataT = TypeVar('FileMetaDataT')

PathLikeType: TypeAlias = str | os.PathLike
Expand Down
1 change: 0 additions & 1 deletion app/Services/storage/s3_compatible_storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from opendal.exceptions import NotFound, PermissionDenied, AlreadyExists
from wcmatch import glob

from app.Models.img_data import ImageData
from app.Services.storage.base import BaseStorage, FileMetaDataT, RemoteFilePathType, LocalFilePathType, \
LocalFileMetaDataType, RemoteFileMetaDataType
from app.Services.storage.exception import LocalFileNotFoundError, RemoteFileNotFoundError, RemoteFilePermissionError, \
Expand Down
4 changes: 1 addition & 3 deletions app/Services/upload_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,13 @@ async def _upload_task(self, img: Image.Image, img_data: ImageData, img_bytes: b
logger.info('Start indexing image {}. Local: {}. Size: {}', img_data.id, img_data.local, len(img_bytes))
file_name = f"{img_data.id}.{img_data.format}"
thumb_path = f"thumbnails/{img_data.id}.webp"
img_thumb = None
if img_data.local:
img_data.url = await self._storage_service.active_storage.url(file_name)
if len(img_bytes) > 1024 * 500:
img_data.thumbnail_url = await self._storage_service.active_storage.url(
f"thumbnails/{img_data.id}.webp")

await self._index_service.index_image(img, img_data, skip_ocr=skip_ocr,
background=True) # The img might be modified after calling this
await self._index_service.index_image(img, img_data, skip_ocr=skip_ocr, background=True)
logger.success("Image {} indexed.", img_data.id)

if img_data.local:
Expand Down
33 changes: 22 additions & 11 deletions readme.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

[![GitHub Workflow Status (with event)](https://img.shields.io/github/actions/workflow/status/hv0905/NekoImageGallery/prod.yml?logo=github)](https://github.com/hv0905/NekoImageGallery/actions)
[![codecov](https://codecov.io/gh/hv0905/NekoImageGallery/branch/master/graph/badge.svg?token=JK2KZBDIYP)](https://codecov.io/gh/hv0905/NekoImageGallery)
[![Maintainability](https://api.codeclimate.com/v1/badges/ac97a1146648996b68ea/maintainability)](https://codeclimate.com/github/hv0905/NekoImageGallery/maintainability)
![Man hours](https://img.shields.io/endpoint?url=https%3A%2F%2Fmanhours.aiursoft.cn%2Fr%2Fgithub.com%2Fhv0905%2FNekoImageGallery.json)
[![Docker Pulls](https://img.shields.io/docker/pulls/edgeneko/neko-image-gallery)](https://hub.docker.com/r/edgeneko/neko-image-gallery)

An online AI image search engine based on the Clip model and Qdrant vector database. Supports keyword search and similar
image search.
Expand Down Expand Up @@ -117,19 +119,28 @@ Local file storage does not require an additional database deployment process, b
NekoImageGallery's docker image are built and released on Docker Hub, including serval variants:
| Tags | Description | Latest Image Size |
|---------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `edgeneko/neko-image-gallery:<version>`<br>`edgeneko/neko-image-gallery:<version>-cuda`<br>`edgeneko/neko-image-gallery:<version>-cuda12.1` | Supports GPU inferencing with CUDA12.1 | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/edgeneko/neko-image-gallery/latest?label=Docker%20Image%20(cuda))](https://hub.docker.com/r/edgeneko/neko-image-gallery) |
| `edgeneko/neko-image-gallery:<version>-cuda11.8` | Supports GPU inferencing with CUDA11.8 | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/edgeneko/neko-image-gallery/latest-cuda11.8?label=Docker%20Image%20(cuda11.8))](https://hub.docker.com/r/edgeneko/neko-image-gallery) |
| `edgeneko/neko-image-gallery:<version>-cpu` | Only supports CPU inferencing | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/edgeneko/neko-image-gallery/latest-cpu?label=Docker%20Image%20(cpu))](https://hub.docker.com/r/edgeneko/neko-image-gallery) |
| Tags | Description | Latest Image Size |
|---------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `edgeneko/neko-image-gallery:<version>`<br>`edgeneko/neko-image-gallery:<version>-cuda`<br>`edgeneko/neko-image-gallery:<version>-cuda12.1` | Supports GPU inferencing with CUDA12.1 | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/edgeneko/neko-image-gallery/latest?label=Image%20(cuda))](https://hub.docker.com/r/edgeneko/neko-image-gallery) |
| `edgeneko/neko-image-gallery:<version>-cuda11.8` | Supports GPU inferencing with CUDA11.8 | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/edgeneko/neko-image-gallery/latest-cuda11.8?label=Image%20(cuda11.8))](https://hub.docker.com/r/edgeneko/neko-image-gallery) |
| `edgeneko/neko-image-gallery:<version>-cpu` | Only supports CPU inferencing | [![Docker Image Size (tag)](https://img.shields.io/docker/image-size/edgeneko/neko-image-gallery/latest-cpu?label=Image%20(cpu))](https://hub.docker.com/r/edgeneko/neko-image-gallery) |
Where `<version>` is the version number or version alias of NekoImageGallery, as follows:
| Version | Description |
|----------|--------------------------------------------------------------------------------------------------------|
| `latest` | The latest stable version of NekoImageGallery |
| `v0.1.0` | The specific version number (correspond to Git tags) |
| `edge` | The latest development version of NekoImageGallery, may contain unstable features and breaking changes |
| Version | Description |
|-------------------|--------------------------------------------------------------------------------------------------------|
| `latest` | The latest stable version of NekoImageGallery |
| `v*.*.*` / `v*.*` | The specific version number (correspond to Git tags) |
| `edge` | The latest development version of NekoImageGallery, may contain unstable features and breaking changes |
In each image, we have bundled the necessary dependencies, `openai/clip-vit-large-patch14` model
weights, `bert-base-chinese` model weights and `easy-paddle-ocr` models to provide a complete and ready-to-use image.
The images uses `/opt/NekoImageGallery/static` as volume to store image files, mount it to your own volume or directory
if local storage is required.
For configuration, we suggest using environment variables to override the default configuration. Secrets (like API
tokens) can be provided by [docker secrets](https://docs.docker.com/engine/swarm/secrets/).
#### Prepare `nvidia-container-runtime` (CUDA users only)
Expand Down Expand Up @@ -192,4 +203,4 @@ define them more clearly.
Copyright 2023 EdgeNeko
Licensed under GPLv3 license.
Licensed under AGPLv3 license.
Loading

0 comments on commit efd3d75

Please sign in to comment.