Skip to content

Commit

Permalink
feat: PDF segmentation - viewer uses 4x less memory on mobile (#254)
Browse files Browse the repository at this point in the history 
* feat(conversion): add mobile optimization to PDF pipeline

* feat: working implementation with API endpoints

* fix(api): OpenAPI docs

* chore: revert unnecessary changes

* fix(api): OpenAPI docs

* docs: update DEVELOPMENT.md

* chore: rename to segmentation

* chore: update OpenAPI docs

* fix: revert DTO changes

* chore: update OpenAPI docs

* fix(migrations): file header

* chore(ui): add PDFProps field

* feat: use ImageMagick to generate PDF thumbnails as JPEG

* chore: generate OpenAPI docs
  • Loading branch information
@bouassaba
bouassaba authored Aug 9, 2024
1 parent 1d8b54e commit bb97963
Show file tree
Hide file tree
Showing 19 changed files with 779 additions and 154 deletions.
1 change: 1 addition & 0 deletions DEVELOPMENT.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ brew install --cask libreoffice
```shell
brew install \
ocrmypdf \
qpdf \
exiftool \
poppler \
imagemagick \
Expand Down
92 changes: 58 additions & 34 deletions api/docs/index.html
View file

Large diffs are not rendered by default.

112 changes: 102 additions & 10 deletions api/docs/swagger.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,11 @@ definitions:
width:
type: integer
type: object
model.PDFProps:
properties:
pages:
type: integer
type: object
model.S3Object:
properties:
bucket:
Expand All @@ -95,6 +100,8 @@ definitions:
$ref: '#/definitions/model.ImageProps'
key:
type: string
pdf:
$ref: '#/definitions/model.PDFProps'
size:
type: integer
type: object
Expand Down Expand Up @@ -220,6 +227,8 @@ definitions:
type: string
image:
$ref: '#/definitions/service.ImageProps'
pdf:
$ref: '#/definitions/service.PDFProps'
size:
type: integer
type: object
Expand Down Expand Up @@ -538,6 +547,11 @@ definitions:
totalPages:
type: integer
type: object
service.PDFProps:
properties:
pages:
type: integer
type: object
service.Snapshot:
properties:
createTime:
Expand All @@ -558,6 +572,8 @@ definitions:
$ref: '#/definitions/service.Download'
preview:
$ref: '#/definitions/service.Download'
segmentation:
$ref: '#/definitions/service.Download'
status:
type: string
task:
Expand Down Expand Up @@ -618,6 +634,8 @@ definitions:
$ref: '#/definitions/model.S3Object'
preview:
$ref: '#/definitions/model.S3Object'
segmentation:
$ref: '#/definitions/model.S3Object'
status:
type: string
taskId:
Expand Down Expand Up @@ -669,8 +687,14 @@ definitions:
type: boolean
name:
type: string
payload:
additionalProperties:
type: string
type: object
percentage:
type: integer
status:
type: string
userId:
type: string
type: object
Expand Down Expand Up @@ -1345,14 +1369,14 @@ paths:
name: id
required: true
type: string
- description: Access Token
in: query
name: access_token
- description: Extension
in: path
name: ext
required: true
type: string
- description: Extension
- description: Access Token
in: query
name: ext
name: access_token
required: true
type: string
produces:
Expand Down Expand Up @@ -1399,6 +1423,74 @@ paths:
summary: Revoke Group Permission
tags:
- Files
/files/{id}/segmentation/pages/{page}.png:
get:
description: Download Segmentation Page
operationId: files_download_segmentation_page
parameters:
- description: ID
in: path
name: id
required: true
type: string
- description: Page
in: path
name: page
required: true
type: string
- description: Access Token
in: query
name: access_token
required: true
type: string
produces:
- application/json
responses:
"404":
description: Not Found
schema:
$ref: '#/definitions/errorpkg.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/errorpkg.ErrorResponse'
summary: Download Segmentation Page
tags:
- Files
/files/{id}/segmentation/thumbnails/{page}.jpg:
get:
description: Download Segmentation Thumbnail
operationId: files_download_segmentation_thumbnail
parameters:
- description: ID
in: path
name: id
required: true
type: string
- description: Page
in: path
name: page
required: true
type: string
- description: Access Token
in: query
name: access_token
required: true
type: string
produces:
- application/json
responses:
"404":
description: Not Found
schema:
$ref: '#/definitions/errorpkg.ErrorResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/errorpkg.ErrorResponse'
summary: Download Segmentation Thumbnail
tags:
- Files
/files/{id}/size:
get:
description: Get Size
Expand Down Expand Up @@ -1437,14 +1529,14 @@ paths:
name: id
required: true
type: string
- description: Access Token
in: query
name: access_token
- description: Extension
in: path
name: ext
required: true
type: string
- description: Extension
- description: Access Token
in: query
name: ext
name: access_token
required: true
type: string
produces:
Expand Down
8 changes: 8 additions & 0 deletions api/model/snapshot_model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type Snapshot interface {
GetOCR() *S3Object
GetEntities() *S3Object
GetMosaic() *S3Object
GetSegmentation() *S3Object
GetThumbnail() *S3Object
GetTaskID() *string
HasOriginal() bool
Expand All @@ -34,6 +35,7 @@ type Snapshot interface {
HasOCR() bool
HasEntities() bool
HasMosaic() bool
HasSegmentation() bool
HasThumbnail() bool
GetStatus() string
GetLanguage() *string
Expand All @@ -47,6 +49,7 @@ type Snapshot interface {
SetOCR(*S3Object)
SetEntities(*S3Object)
SetMosaic(*S3Object)
SetSegmentation(*S3Object)
SetThumbnail(*S3Object)
SetStatus(string)
SetLanguage(string)
Expand All @@ -58,13 +61,18 @@ type S3Object struct {
Key string `json:"key"`
Size *int64 `json:"size,omitempty"`
Image *ImageProps `json:"image,omitempty"`
PDF *PDFProps `json:"pdf,omitempty"`
}

type ImageProps struct {
Width int `json:"width"`
Height int `json:"height"`
}

type PDFProps struct {
Pages int `json:"pages"`
}

type S3Reference struct {
Bucket string `json:"bucket"`
Key string `json:"key"`
Expand Down
107 changes: 72 additions & 35 deletions api/repo/snapshot_repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,20 +56,21 @@ func NewSnapshot() model.Snapshot {
}

type snapshotEntity struct {
ID string `gorm:"column:id;size:36" json:"id"`
Version int64 `gorm:"column:version" json:"version"`
Original datatypes.JSON `gorm:"column:original" json:"original,omitempty"`
Preview datatypes.JSON `gorm:"column:preview" json:"preview,omitempty"`
Text datatypes.JSON `gorm:"column:text" json:"text,omitempty"`
OCR datatypes.JSON `gorm:"column:ocr" json:"ocr,omitempty"`
Entities datatypes.JSON `gorm:"column:entities" json:"entities,omitempty"`
Mosaic datatypes.JSON `gorm:"column:mosaic" json:"mosaic,omitempty"`
Thumbnail datatypes.JSON `gorm:"column:thumbnail" json:"thumbnail,omitempty"`
Status string `gorm:"column,status" json:"status,omitempty"`
Language *string `gorm:"column:language" json:"language,omitempty"`
TaskID *string `gorm:"column:task_id" json:"taskId,omitempty"`
CreateTime string `gorm:"column:create_time" json:"createTime"`
UpdateTime *string `gorm:"column:update_time" json:"updateTime,omitempty"`
ID string `gorm:"column:id;size:36" json:"id"`
Version int64 `gorm:"column:version" json:"version"`
Original datatypes.JSON `gorm:"column:original" json:"original,omitempty"`
Preview datatypes.JSON `gorm:"column:preview" json:"preview,omitempty"`
Text datatypes.JSON `gorm:"column:text" json:"text,omitempty"`
OCR datatypes.JSON `gorm:"column:ocr" json:"ocr,omitempty"`
Entities datatypes.JSON `gorm:"column:entities" json:"entities,omitempty"`
Mosaic datatypes.JSON `gorm:"column:mosaic" json:"mosaic,omitempty"`
Segmentation datatypes.JSON `gorm:"column:segmentation" json:"segmentation,omitempty"`
Thumbnail datatypes.JSON `gorm:"column:thumbnail" json:"thumbnail,omitempty"`
Status string `gorm:"column,status" json:"status,omitempty"`
Language *string `gorm:"column:language" json:"language,omitempty"`
TaskID *string `gorm:"column:task_id" json:"taskId,omitempty"`
CreateTime string `gorm:"column:create_time" json:"createTime"`
UpdateTime *string `gorm:"column:update_time" json:"updateTime,omitempty"`
}

func (*snapshotEntity) TableName() string {
Expand Down Expand Up @@ -167,6 +168,18 @@ func (s *snapshotEntity) GetMosaic() *model.S3Object {
return &res
}

func (s *snapshotEntity) GetSegmentation() *model.S3Object {
if s.Segmentation.String() == "" {
return nil
}
res := model.S3Object{}
if err := json.Unmarshal([]byte(s.Segmentation.String()), &res); err != nil {
log.GetLogger().Fatal(err)
return nil
}
return &res
}

func (s *snapshotEntity) GetThumbnail() *model.S3Object {
if s.Thumbnail.String() == "" {
return nil
Expand Down Expand Up @@ -289,6 +302,21 @@ func (s *snapshotEntity) SetMosaic(m *model.S3Object) {
}
}

func (s *snapshotEntity) SetSegmentation(m *model.S3Object) {
if m == nil {
s.Segmentation = nil
} else {
b, err := json.Marshal(m)
if err != nil {
log.GetLogger().Fatal(err)
return
}
if err := s.Segmentation.UnmarshalJSON(b); err != nil {
log.GetLogger().Fatal(err)
}
}
}

func (s *snapshotEntity) SetThumbnail(m *model.S3Object) {
if m == nil {
s.Thumbnail = nil
Expand Down Expand Up @@ -340,6 +368,10 @@ func (s *snapshotEntity) HasMosaic() bool {
return s.Mosaic != nil
}

func (s *snapshotEntity) HasSegmentation() bool {
return s.Segmentation != nil
}

func (s *snapshotEntity) HasThumbnail() bool {
return s.Thumbnail != nil
}
Expand Down Expand Up @@ -436,30 +468,32 @@ func (repo *snapshotRepo) Delete(id string) error {
}

type SnapshotUpdateOptions struct {
Fields []string `json:"fields"`
Original *model.S3Object
Preview *model.S3Object
Text *model.S3Object
OCR *model.S3Object
Entities *model.S3Object
Mosaic *model.S3Object
Thumbnail *model.S3Object
Status *string
Language *string
TaskID *string
Fields []string `json:"fields"`
Original *model.S3Object
Preview *model.S3Object
Text *model.S3Object
OCR *model.S3Object
Entities *model.S3Object
Mosaic *model.S3Object
Segmentation *model.S3Object
Thumbnail *model.S3Object
Status *string
Language *string
TaskID *string
}

const (
SnapshotFieldOriginal = "original"
SnapshotFieldPreview = "preview"
SnapshotFieldText = "text"
SnapshotFieldOCR = "ocr"
SnapshotFieldEntities = "entities"
SnapshotFieldMosaic = "mosaic"
SnapshotFieldThumbnail = "thumbnail"
SnapshotFieldStatus = "status"
SnapshotFieldLanguage = "language"
SnapshotFieldTaskID = "taskId"
SnapshotFieldOriginal = "original"
SnapshotFieldPreview = "preview"
SnapshotFieldText = "text"
SnapshotFieldOCR = "ocr"
SnapshotFieldEntities = "entities"
SnapshotFieldMosaic = "mosaic"
SnapshotFieldSegmentation = "segmentation"
SnapshotFieldThumbnail = "thumbnail"
SnapshotFieldStatus = "status"
SnapshotFieldLanguage = "language"
SnapshotFieldTaskID = "taskId"
)

func (repo *snapshotRepo) Update(id string, opts SnapshotUpdateOptions) error {
Expand All @@ -485,6 +519,9 @@ func (repo *snapshotRepo) Update(id string, opts SnapshotUpdateOptions) error {
if slices.Contains(opts.Fields, SnapshotFieldMosaic) {
snapshot.SetMosaic(opts.Mosaic)
}
if slices.Contains(opts.Fields, SnapshotFieldSegmentation) {
snapshot.SetSegmentation(opts.Segmentation)
}
if slices.Contains(opts.Fields, SnapshotFieldThumbnail) {
snapshot.SetThumbnail(opts.Thumbnail)
}
Expand Down
Loading

0 comments on commit bb97963

Please sign in to comment.