diff --git a/README.md b/README.md index 6c46166..c70b95c 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,25 @@ So, for example, volume "2" would be placed before "10", while "02" would be cor kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --fill-volume-number 2 ``` +### Use lower quality images to save space + +Kojirou has the ability to download lower-quality images from MangaDex. +This can be useful to save space on your device, or to reduce the amount of data downloaded on slow or limited connections. +Legal arguments to this option are "no", "prefer" and "fallback". + +``` +kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=prefer +``` + +### Fallback to lower quality alternatives for broken images + +MangaDex sometimes hosts images that are subtly broken and cannot be reliably converted to an image format compatible with Kindle devices. +Kojirou can be configured to fall back on reencoded lower-quality versions of these images, which often do not have the same problems. + +``` +kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=fallback +``` + ## Prebuilt binaries Prebuilt binaries for Linux, Windows and MacOS on x86 and ARM processors are provided. diff --git a/cmd/business.go b/cmd/business.go index e0232e6..d2681bb 100644 --- a/cmd/business.go +++ b/cmd/business.go @@ -143,7 +143,7 @@ func getCovers(manga *md.Manga) (md.ImageList, error) { func getPages(volume md.Volume, p formats.CliProgress) (md.ImageList, error) { mangadexPages, err := download.MangadexPages(volume.Sorted().FilterBy(func(ci md.ChapterInfo) bool { return ci.GroupNames.String() != "Filesystem" - }), p) + }), dataSaverArg, p) if err != nil { p.Cancel("Error") return nil, fmt.Errorf("mangadex: %w", err) diff --git a/cmd/formats/download/policy.go b/cmd/formats/download/policy.go new file mode 100644 index 0000000..a7e4e6a --- /dev/null +++ b/cmd/formats/download/policy.go @@ -0,0 +1,45 @@ +package download + +import "fmt" + +type DataSaverPolicy int + +const ( + DataSaverPolicyNo DataSaverPolicy = iota + DataSaverPolicyPrefer + DataSaverPolicyFallback +) + +func (p *DataSaverPolicy) String() string { + switch *p { + case DataSaverPolicyNo: + return "no" + case DataSaverPolicyPrefer: + return "prefer" + case DataSaverPolicyFallback: + return "fallback" + default: + panic("unreachable") + } +} + +// Set must have pointer receiver so it doesn't change the value of a copy +func (p *DataSaverPolicy) Set(v string) error { + switch v { + case "no": + *p = DataSaverPolicyNo + case "prefer": + *p = DataSaverPolicyPrefer + case "fallback": + *p = DataSaverPolicyFallback + default: + return fmt.Errorf(`must be one of: "no", "prefer", or "fallback"`) + } + + return nil +} + +// Type is only used in help text +func (p *DataSaverPolicy) Type() string { + return "data-saver policy" +} diff --git a/cmd/formats/download/root.go b/cmd/formats/download/root.go index 205c10d..e9c0f53 100644 --- a/cmd/formats/download/root.go +++ b/cmd/formats/download/root.go @@ -61,7 +61,7 @@ func MangadexCovers(manga *md.Manga, p formats.Progress) (md.ImageList, error) { close(coverPaths) }() - coverImages, eg := pathsToImages(coverPaths, ctx, cancel) + coverImages, eg := pathsToImages(coverPaths, ctx, cancel, DataSaverPolicyNo) results := make(md.ImageList, len(covers)) for coverImage := range coverImages { @@ -76,7 +76,7 @@ func MangadexCovers(manga *md.Manga, p formats.Progress) (md.ImageList, error) { } } -func MangadexPages(chapterList md.ChapterList, p formats.Progress) (md.ImageList, error) { +func MangadexPages(chapterList md.ChapterList, policy DataSaverPolicy, p formats.Progress) (md.ImageList, error) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() @@ -94,7 +94,7 @@ func MangadexPages(chapterList md.ChapterList, p formats.Progress) (md.ImageList paths, childEg := chaptersToPaths(chapters, ctx, cancel, p) eg.Go(childEg.Wait) - images, childEg := pathsToImages(paths, ctx, cancel) + images, childEg := pathsToImages(paths, ctx, cancel, policy) eg.Go(childEg.Wait) results := make(md.ImageList, 0) @@ -163,6 +163,7 @@ func pathsToImages( paths <-chan md.Path, ctx context.Context, cancel context.CancelFunc, + policy DataSaverPolicy, ) (<-chan md.Image, *errgroup.Group) { ch := make(chan md.Image) eg, ctx := errgroup.WithContext(ctx) @@ -178,17 +179,17 @@ func pathsToImages( return nil } eg.Go(func() error { - image, err := getImage(httpClient, ctx, path.URL) + img, err := getImageWithPolicy(httpClient, ctx, path, policy) if err != nil { defer cancel() return fmt.Errorf("chapter %v: image %v: %w", path.ChapterIdentifier, path.ImageIdentifier, err) - } else { - select { - case <-ctx.Done(): - return fmt.Errorf("canceled") - case ch <- path.WithImage(image): - return nil - } + } + + select { + case <-ctx.Done(): + return fmt.Errorf("canceled") + case ch <- path.WithImage(img): + return nil } }) } @@ -203,7 +204,34 @@ func pathsToImages( return ch, eg } -func getImage(client *http.Client, ctx context.Context, url string) (image.Image, error) { +func getImageWithPolicy(client *http.Client, ctx context.Context, path md.Path, policy DataSaverPolicy) (image.Image, error) { + resp := new(http.Response) + err := error(nil) + + switch policy { + case DataSaverPolicyNo, DataSaverPolicyFallback: + resp, err = getResp(httpClient, ctx, path.DataURL) + case DataSaverPolicyPrefer: + resp, err = getResp(httpClient, ctx, path.DataSaverURL) + } + + if err != nil { + return nil, fmt.Errorf("download: %w", err) + } + + img, _, err := image.Decode(resp.Body) + defer resp.Body.Close() + + if err != nil && policy == DataSaverPolicyFallback { + return getImageWithPolicy(client, ctx, path, DataSaverPolicyPrefer) + } else if err != nil { + return nil, fmt.Errorf("decode: %w", err) + } else { + return img, nil + } +} + +func getResp(client *http.Client, ctx context.Context, url string) (*http.Response, error) { req, err := http.NewRequestWithContext(ctx, "GET", url, nil) if err != nil { return nil, fmt.Errorf("prepare: %w", err) @@ -212,15 +240,10 @@ func getImage(client *http.Client, ctx context.Context, url string) (image.Image if err != nil { return nil, fmt.Errorf("do: %w", err) } - defer resp.Body.Close() if resp.StatusCode != 200 { return nil, fmt.Errorf("status: %v", resp.Status) } - img, _, err := image.Decode(resp.Body) - if err != nil { - return nil, fmt.Errorf("decode: %w", err) - } - return img, nil + return resp, nil } diff --git a/cmd/root.go b/cmd/root.go index 28eb627..a5f3a92 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,6 +4,7 @@ import ( "os" "runtime/pprof" + "github.com/leotaku/kojirou/cmd/formats/download" "github.com/spf13/cobra" ) @@ -18,6 +19,7 @@ var ( forceArg bool leftToRightArg bool fillVolumeNumberArg int + dataSaverArg download.DataSaverPolicy diskArg string cpuprofileArg string groupsFilter string @@ -156,6 +158,7 @@ func init() { rootCmd.Flags().BoolVarP(&kindleFolderModeArg, "kindle-folder-mode", "k", false, "generate folder structure for Kindle devices") rootCmd.Flags().BoolVarP(&leftToRightArg, "left-to-right", "p", false, "make reading direction left to right") rootCmd.Flags().IntVarP(&fillVolumeNumberArg, "fill-volume-number", "n", 0, "fill volume number with leading zeros in title") + rootCmd.Flags().VarP(&dataSaverArg, "data-saver", "s", "download lower quality images to save space") rootCmd.Flags().BoolVarP(&dryRunArg, "dry-run", "d", false, "disable writing of any files") rootCmd.Flags().StringVarP(&outArg, "out", "o", "", "output directory") rootCmd.Flags().BoolVarP(&forceArg, "force", "f", false, "overwrite existing volumes") diff --git a/mangadex/client.go b/mangadex/client.go index 763bad1..f26725b 100644 --- a/mangadex/client.go +++ b/mangadex/client.go @@ -130,6 +130,8 @@ func (c *Client) FetchPaths(ctx context.Context, chapter *Chapter) (PathList, er ah, err := c.base.GetAtHome(ctx, chapter.Info.ID) if err != nil { return nil, fmt.Errorf("get at home: %w", err) + } else if len(ah.Chapter.Data) != len(ah.Chapter.DataSaver) { + return nil, fmt.Errorf("broken chapter image list") } return convertChapter(chapter, ah), nil diff --git a/mangadex/convert.go b/mangadex/convert.go index c5f9888..97ed5b0 100644 --- a/mangadex/convert.go +++ b/mangadex/convert.go @@ -61,7 +61,7 @@ func convertCovers(coverBaseURL string, mangaID string, co []api.CoverData) Path for _, info := range co { url := strings.Join([]string{coverBaseURL, mangaID, info.Attributes.FileName}, "/") result = append(result, Path{ - URL: url, + DataURL: url, ImageIdentifier: 0, ChapterIdentifier: NewIdentifier("0"), VolumeIdentifier: NewWithFallback(info.Attributes.Volume, "Special"), @@ -73,10 +73,12 @@ func convertCovers(coverBaseURL string, mangaID string, co []api.CoverData) Path func convertChapter(ch *Chapter, ah *api.AtHome) PathList { result := make(PathList, 0) - for i, filename := range ah.Chapter.Data { - url := strings.Join([]string{ah.BaseURL, "data", ah.Chapter.Hash, filename}, "/") + for i := range ah.Chapter.Data { + dataURL := strings.Join([]string{ah.BaseURL, "data", ah.Chapter.Hash, ah.Chapter.Data[i]}, "/") + dataSaverURL := strings.Join([]string{ah.BaseURL, "data-saver", ah.Chapter.Hash, ah.Chapter.DataSaver[i]}, "/") result = append(result, Path{ - URL: url, + DataURL: dataURL, + DataSaverURL: dataSaverURL, ImageIdentifier: i, ChapterIdentifier: ch.Info.Identifier, VolumeIdentifier: ch.Info.VolumeIdentifier, diff --git a/mangadex/unstructured.go b/mangadex/unstructured.go index a216a1c..2525f37 100644 --- a/mangadex/unstructured.go +++ b/mangadex/unstructured.go @@ -41,7 +41,8 @@ type Image struct { } type Path struct { - URL string + DataURL string + DataSaverURL string // identifiers ImageIdentifier int