From 2024a8503b46585769ef2d2c2eaad2b5c7dd3e81 Mon Sep 17 00:00:00 2001 From: Leo Gaskin Date: Sun, 9 Apr 2023 22:44:09 +0200 Subject: [PATCH 1/6] Add preliminary support for data-saver to the API client --- cmd/formats/download/root.go | 2 +- mangadex/convert.go | 10 ++++++---- mangadex/unstructured.go | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/formats/download/root.go b/cmd/formats/download/root.go index 205c10d..7e0cfef 100644 --- a/cmd/formats/download/root.go +++ b/cmd/formats/download/root.go @@ -178,7 +178,7 @@ func pathsToImages( return nil } eg.Go(func() error { - image, err := getImage(httpClient, ctx, path.URL) + image, err := getImage(httpClient, ctx, path.DataURL) if err != nil { defer cancel() return fmt.Errorf("chapter %v: image %v: %w", path.ChapterIdentifier, path.ImageIdentifier, err) diff --git a/mangadex/convert.go b/mangadex/convert.go index c5f9888..97ed5b0 100644 --- a/mangadex/convert.go +++ b/mangadex/convert.go @@ -61,7 +61,7 @@ func convertCovers(coverBaseURL string, mangaID string, co []api.CoverData) Path for _, info := range co { url := strings.Join([]string{coverBaseURL, mangaID, info.Attributes.FileName}, "/") result = append(result, Path{ - URL: url, + DataURL: url, ImageIdentifier: 0, ChapterIdentifier: NewIdentifier("0"), VolumeIdentifier: NewWithFallback(info.Attributes.Volume, "Special"), @@ -73,10 +73,12 @@ func convertCovers(coverBaseURL string, mangaID string, co []api.CoverData) Path func convertChapter(ch *Chapter, ah *api.AtHome) PathList { result := make(PathList, 0) - for i, filename := range ah.Chapter.Data { - url := strings.Join([]string{ah.BaseURL, "data", ah.Chapter.Hash, filename}, "/") + for i := range ah.Chapter.Data { + dataURL := strings.Join([]string{ah.BaseURL, "data", ah.Chapter.Hash, ah.Chapter.Data[i]}, "/") + dataSaverURL := strings.Join([]string{ah.BaseURL, "data-saver", ah.Chapter.Hash, ah.Chapter.DataSaver[i]}, "/") result = append(result, Path{ - URL: url, + DataURL: dataURL, + DataSaverURL: dataSaverURL, ImageIdentifier: i, ChapterIdentifier: ch.Info.Identifier, VolumeIdentifier: ch.Info.VolumeIdentifier, diff --git a/mangadex/unstructured.go b/mangadex/unstructured.go index a216a1c..2525f37 100644 --- a/mangadex/unstructured.go +++ b/mangadex/unstructured.go @@ -41,7 +41,8 @@ type Image struct { } type Path struct { - URL string + DataURL string + DataSaverURL string // identifiers ImageIdentifier int From b47729ae8aa70b183fd0f94e37f24d43123f03fd Mon Sep 17 00:00:00 2001 From: Leo Gaskin Date: Sun, 9 Apr 2023 22:51:16 +0200 Subject: [PATCH 2/6] Commit support for new "--data-saver" option --- cmd/business.go | 2 +- cmd/formats/download/policy.go | 45 ++++++++++++++++++++++++++++++++++ cmd/formats/download/root.go | 39 ++++++++++++++++++++--------- cmd/root.go | 3 +++ 4 files changed, 76 insertions(+), 13 deletions(-) create mode 100644 cmd/formats/download/policy.go diff --git a/cmd/business.go b/cmd/business.go index e0232e6..d2681bb 100644 --- a/cmd/business.go +++ b/cmd/business.go @@ -143,7 +143,7 @@ func getCovers(manga *md.Manga) (md.ImageList, error) { func getPages(volume md.Volume, p formats.CliProgress) (md.ImageList, error) { mangadexPages, err := download.MangadexPages(volume.Sorted().FilterBy(func(ci md.ChapterInfo) bool { return ci.GroupNames.String() != "Filesystem" - }), p) + }), dataSaverArg, p) if err != nil { p.Cancel("Error") return nil, fmt.Errorf("mangadex: %w", err) diff --git a/cmd/formats/download/policy.go b/cmd/formats/download/policy.go new file mode 100644 index 0000000..a7e4e6a --- /dev/null +++ b/cmd/formats/download/policy.go @@ -0,0 +1,45 @@ +package download + +import "fmt" + +type DataSaverPolicy int + +const ( + DataSaverPolicyNo DataSaverPolicy = iota + DataSaverPolicyPrefer + DataSaverPolicyFallback +) + +func (p *DataSaverPolicy) String() string { + switch *p { + case DataSaverPolicyNo: + return "no" + case DataSaverPolicyPrefer: + return "prefer" + case DataSaverPolicyFallback: + return "fallback" + default: + panic("unreachable") + } +} + +// Set must have pointer receiver so it doesn't change the value of a copy +func (p *DataSaverPolicy) Set(v string) error { + switch v { + case "no": + *p = DataSaverPolicyNo + case "prefer": + *p = DataSaverPolicyPrefer + case "fallback": + *p = DataSaverPolicyFallback + default: + return fmt.Errorf(`must be one of: "no", "prefer", or "fallback"`) + } + + return nil +} + +// Type is only used in help text +func (p *DataSaverPolicy) Type() string { + return "data-saver policy" +} diff --git a/cmd/formats/download/root.go b/cmd/formats/download/root.go index 7e0cfef..1200459 100644 --- a/cmd/formats/download/root.go +++ b/cmd/formats/download/root.go @@ -61,7 +61,7 @@ func MangadexCovers(manga *md.Manga, p formats.Progress) (md.ImageList, error) { close(coverPaths) }() - coverImages, eg := pathsToImages(coverPaths, ctx, cancel) + coverImages, eg := pathsToImages(coverPaths, ctx, cancel, DataSaverPolicyNo) results := make(md.ImageList, len(covers)) for coverImage := range coverImages { @@ -76,7 +76,7 @@ func MangadexCovers(manga *md.Manga, p formats.Progress) (md.ImageList, error) { } } -func MangadexPages(chapterList md.ChapterList, p formats.Progress) (md.ImageList, error) { +func MangadexPages(chapterList md.ChapterList, policy DataSaverPolicy, p formats.Progress) (md.ImageList, error) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() @@ -94,7 +94,7 @@ func MangadexPages(chapterList md.ChapterList, p formats.Progress) (md.ImageList paths, childEg := chaptersToPaths(chapters, ctx, cancel, p) eg.Go(childEg.Wait) - images, childEg := pathsToImages(paths, ctx, cancel) + images, childEg := pathsToImages(paths, ctx, cancel, policy) eg.Go(childEg.Wait) results := make(md.ImageList, 0) @@ -163,6 +163,7 @@ func pathsToImages( paths <-chan md.Path, ctx context.Context, cancel context.CancelFunc, + policy DataSaverPolicy, ) (<-chan md.Image, *errgroup.Group) { ch := make(chan md.Image) eg, ctx := errgroup.WithContext(ctx) @@ -178,17 +179,31 @@ func pathsToImages( return nil } eg.Go(func() error { - image, err := getImage(httpClient, ctx, path.DataURL) + image := image.Image(nil) + err := error(nil) + + switch policy { + case DataSaverPolicyNo: + image, err = getImage(httpClient, ctx, path.DataURL) + case DataSaverPolicyPrefer: + image, err = getImage(httpClient, ctx, path.DataSaverURL) + case DataSaverPolicyFallback: + image, err = getImage(httpClient, ctx, path.DataURL) + if err != nil && policy == DataSaverPolicyFallback { + image, err = getImage(httpClient, ctx, path.DataSaverURL) + } + } + if err != nil { - defer cancel() + cancel() return fmt.Errorf("chapter %v: image %v: %w", path.ChapterIdentifier, path.ImageIdentifier, err) - } else { - select { - case <-ctx.Done(): - return fmt.Errorf("canceled") - case ch <- path.WithImage(image): - return nil - } + } + + select { + case <-ctx.Done(): + return fmt.Errorf("canceled") + case ch <- path.WithImage(image): + return nil } }) } diff --git a/cmd/root.go b/cmd/root.go index 28eb627..a5f3a92 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ import ( "os" "runtime/pprof" + "github.com/leotaku/kojirou/cmd/formats/download" "github.com/spf13/cobra" ) @@ -18,6 +19,7 @@ var ( forceArg bool leftToRightArg bool fillVolumeNumberArg int + dataSaverArg download.DataSaverPolicy diskArg string cpuprofileArg string groupsFilter string @@ -156,6 +158,7 @@ func init() { rootCmd.Flags().BoolVarP(&kindleFolderModeArg, "kindle-folder-mode", "k", false, "generate folder structure for Kindle devices") rootCmd.Flags().BoolVarP(&leftToRightArg, "left-to-right", "p", false, "make reading direction left to right") rootCmd.Flags().IntVarP(&fillVolumeNumberArg, "fill-volume-number", "n", 0, "fill volume number with leading zeros in title") + rootCmd.Flags().VarP(&dataSaverArg, "data-saver", "s", "download lower quality images to save space") rootCmd.Flags().BoolVarP(&dryRunArg, "dry-run", "d", false, "disable writing of any files") rootCmd.Flags().StringVarP(&outArg, "out", "o", "", "output directory") rootCmd.Flags().BoolVarP(&forceArg, "force", "f", false, "overwrite existing volumes") From 289f976d269c2a7c303f3305efb617fc1db62418 Mon Sep 17 00:00:00 2001 From: Leo Gaskin Date: Sun, 9 Apr 2023 22:51:47 +0200 Subject: [PATCH 3/6] Document new "--data-saver" option in README --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index 6c46166..4c9a86f 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,25 @@ So, for example, volume "2" would be placed before "10", while "02" would be cor kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --fill-volume-number 2 ``` +### Use lower quality images to save space + +Kojirou has the ability to download lower-quality images from MangaDex. +This can be useful to save space on your device, or to reduce the amount of data downloaded on slow or limited connections. +Legal arguments to this option are "no", "prefer" and "fallback". + +``` +kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=prefer +``` + +### Fallback to lower quality alternatives for broken images + +MangaDex sometimes hosts images that are broken and can not be reliably converted to a format compatible with Kindle e-books. +Kojirou can be configured to fall back on reencoded lower-quality versions of these images, which are unlikely to also be broken. + +``` +kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=fallback +``` + ## Prebuilt binaries Prebuilt binaries for Linux, Windows and MacOS on x86 and ARM processors are provided. From ac68d89d8035522453acee2511896ab4e51c7255 Mon Sep 17 00:00:00 2001 From: Leo Gaskin Date: Fri, 14 Apr 2023 21:53:38 +0200 Subject: [PATCH 4/6] Ensure no panic is raised when data-saver entries are missing --- mangadex/client.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mangadex/client.go b/mangadex/client.go index 763bad1..f26725b 100644 --- a/mangadex/client.go +++ b/mangadex/client.go @@ -130,6 +130,8 @@ func (c *Client) FetchPaths(ctx context.Context, chapter *Chapter) (PathList, er ah, err := c.base.GetAtHome(ctx, chapter.Info.ID) if err != nil { return nil, fmt.Errorf("get at home: %w", err) + } else if len(ah.Chapter.Data) != len(ah.Chapter.DataSaver) { + return nil, fmt.Errorf("broken chapter image list") } return convertChapter(chapter, ah), nil From c5787e226738e6df3bed7c4e0f26748becd986c4 Mon Sep 17 00:00:00 2001 From: Leo Gaskin Date: Fri, 14 Apr 2023 23:14:50 +0200 Subject: [PATCH 5/6] Factor out policy handling into its own function --- cmd/formats/download/root.go | 56 ++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/cmd/formats/download/root.go b/cmd/formats/download/root.go index 1200459..e9c0f53 100644 --- a/cmd/formats/download/root.go +++ b/cmd/formats/download/root.go @@ -179,30 +179,16 @@ func pathsToImages( return nil } eg.Go(func() error { - image := image.Image(nil) - err := error(nil) - - switch policy { - case DataSaverPolicyNo: - image, err = getImage(httpClient, ctx, path.DataURL) - case DataSaverPolicyPrefer: - image, err = getImage(httpClient, ctx, path.DataSaverURL) - case DataSaverPolicyFallback: - image, err = getImage(httpClient, ctx, path.DataURL) - if err != nil && policy == DataSaverPolicyFallback { - image, err = getImage(httpClient, ctx, path.DataSaverURL) - } - } - + img, err := getImageWithPolicy(httpClient, ctx, path, policy) if err != nil { - cancel() + defer cancel() return fmt.Errorf("chapter %v: image %v: %w", path.ChapterIdentifier, path.ImageIdentifier, err) } select { case <-ctx.Done(): return fmt.Errorf("canceled") - case ch <- path.WithImage(image): + case ch <- path.WithImage(img): return nil } }) @@ -218,7 +204,34 @@ func pathsToImages( return ch, eg } -func getImage(client *http.Client, ctx context.Context, url string) (image.Image, error) { +func getImageWithPolicy(client *http.Client, ctx context.Context, path md.Path, policy DataSaverPolicy) (image.Image, error) { + resp := new(http.Response) + err := error(nil) + + switch policy { + case DataSaverPolicyNo, DataSaverPolicyFallback: + resp, err = getResp(httpClient, ctx, path.DataURL) + case DataSaverPolicyPrefer: + resp, err = getResp(httpClient, ctx, path.DataSaverURL) + } + + if err != nil { + return nil, fmt.Errorf("download: %w", err) + } + + img, _, err := image.Decode(resp.Body) + defer resp.Body.Close() + + if err != nil && policy == DataSaverPolicyFallback { + return getImageWithPolicy(client, ctx, path, DataSaverPolicyPrefer) + } else if err != nil { + return nil, fmt.Errorf("decode: %w", err) + } else { + return img, nil + } +} + +func getResp(client *http.Client, ctx context.Context, url string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return nil, fmt.Errorf("prepare: %w", err) @@ -227,15 +240,10 @@ func getImage(client *http.Client, ctx context.Context, url string) (image.Image if err != nil { return nil, fmt.Errorf("do: %w", err) } - defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("status: %v", resp.Status) } - img, _, err := image.Decode(resp.Body) - if err != nil { - return nil, fmt.Errorf("decode: %w", err) - } - return img, nil + return resp, nil } From e28b60c591322047fd9f27da266abad7165c48ef Mon Sep 17 00:00:00 2001 From: Leo Gaskin Date: Sun, 23 Apr 2023 19:02:52 +0200 Subject: [PATCH 6/6] Improve README description of data-saver feature --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c9a86f..c70b95c 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,8 @@ kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=prefer ### Fallback to lower quality alternatives for broken images -MangaDex sometimes hosts images that are broken and can not be reliably converted to a format compatible with Kindle e-books. -Kojirou can be configured to fall back on reencoded lower-quality versions of these images, which are unlikely to also be broken. +MangaDex sometimes hosts images that are subtly broken and cannot be reliably converted to an image format compatible with Kindle devices. +Kojirou can be configured to fall back on reencoded lower-quality versions of these images, which often do not have the same problems. ``` kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=fallback