Merge pull request #29 from leotaku/data-saver

Implement: Optionally use Data-Saver

README.md

@@ -94,6 +94,25 @@ So, for example, volume "2" would be placed before "10", while "02" would be cor
 kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --fill-volume-number 2
 ```
 
+### Use lower quality images to save space
+
+Kojirou has the ability to download lower-quality images from MangaDex.
+This can be useful to save space on your device, or to reduce the amount of data downloaded on slow or limited connections.
+Legal arguments to this option are "no", "prefer" and "fallback".
+
+```
+kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=prefer
+```
+
+### Fallback to lower quality alternatives for broken images
+
+MangaDex sometimes hosts images that are subtly broken and cannot be reliably converted to an image format compatible with Kindle devices.
+Kojirou can be configured to fall back on reencoded lower-quality versions of these images, which often do not have the same problems.
+
+```
+kojirou d86cf65b-5f6c-437d-a0af-19a31f94ec55 -l en --data-saver=fallback
+```
+
 ## Prebuilt binaries
 
 Prebuilt binaries for Linux, Windows and MacOS on x86 and ARM processors are provided.

cmd/business.go

@@ -143,7 +143,7 @@ func getCovers(manga *md.Manga) (md.ImageList, error) {
 func getPages(volume md.Volume, p formats.CliProgress) (md.ImageList, error) {
 	mangadexPages, err := download.MangadexPages(volume.Sorted().FilterBy(func(ci md.ChapterInfo) bool {
 		return ci.GroupNames.String() != "Filesystem"
-	}), p)
+	}), dataSaverArg, p)
 	if err != nil {
 		p.Cancel("Error")
 		return nil, fmt.Errorf("mangadex: %w", err)

cmd/formats/download/policy.go

@@ -0,0 +1,45 @@
+package download
+
+import "fmt"
+
+type DataSaverPolicy int
+
+const (
+	DataSaverPolicyNo DataSaverPolicy = iota
+	DataSaverPolicyPrefer
+	DataSaverPolicyFallback
+)
+
+func (p *DataSaverPolicy) String() string {
+	switch *p {
+	case DataSaverPolicyNo:
+		return "no"
+	case DataSaverPolicyPrefer:
+		return "prefer"
+	case DataSaverPolicyFallback:
+		return "fallback"
+	default:
+		panic("unreachable")
+	}
+}
+
+// Set must have pointer receiver so it doesn't change the value of a copy
+func (p *DataSaverPolicy) Set(v string) error {
+	switch v {
+	case "no":
+		*p = DataSaverPolicyNo
+	case "prefer":
+		*p = DataSaverPolicyPrefer
+	case "fallback":
+		*p = DataSaverPolicyFallback
+	default:
+		return fmt.Errorf(`must be one of: "no", "prefer", or "fallback"`)
+	}
+
+	return nil
+}
+
+// Type is only used in help text
+func (p *DataSaverPolicy) Type() string {
+	return "data-saver policy"
+}

cmd/formats/download/root.go

@@ -64,7 +64,7 @@ func MangadexCovers(manga *md.Manga, p formats.Progress) (md.ImageList, error) {
 		close(coverPaths)
 	}()
 
-	coverImages, eg := pathsToImages(coverPaths, ctx, cancel)
+	coverImages, eg := pathsToImages(coverPaths, ctx, cancel, DataSaverPolicyNo)
 
 	results := make(md.ImageList, len(covers))
 	for coverImage := range coverImages {
@@ -79,7 +79,7 @@ func MangadexCovers(manga *md.Manga, p formats.Progress) (md.ImageList, error) {
 	}
 }
 
-func MangadexPages(chapterList md.ChapterList, p formats.Progress) (md.ImageList, error) {
+func MangadexPages(chapterList md.ChapterList, policy DataSaverPolicy, p formats.Progress) (md.ImageList, error) {
 	ctx, cancel := context.WithCancel(context.TODO())
 	defer cancel()
 
@@ -97,7 +97,7 @@ func MangadexPages(chapterList md.ChapterList, p formats.Progress) (md.ImageList
 	paths, childEg := chaptersToPaths(chapters, ctx, cancel, p)
 	eg.Go(childEg.Wait)
 
-	images, childEg := pathsToImages(paths, ctx, cancel)
+	images, childEg := pathsToImages(paths, ctx, cancel, policy)
 	eg.Go(childEg.Wait)
 
 	results := make(md.ImageList, 0)
@@ -166,6 +166,7 @@ func pathsToImages(
 	paths <-chan md.Path,
 	ctx context.Context,
 	cancel context.CancelFunc,
+	policy DataSaverPolicy,
 ) (<-chan md.Image, *errgroup.Group) {
 	ch := make(chan md.Image)
 	eg, ctx := errgroup.WithContext(ctx)
@@ -181,17 +182,17 @@ func pathsToImages(
 					return nil
 				}
 				eg.Go(func() error {
-					image, err := getImage(httpClient, ctx, path.URL)
+					img, err := getImageWithPolicy(httpClient, ctx, path, policy)
 					if err != nil {
 						defer cancel()
 						return fmt.Errorf("chapter %v: image %v: %w", path.ChapterIdentifier, path.ImageIdentifier, err)
-					} else {
-						select {
-						case <-ctx.Done():
-							return fmt.Errorf("canceled")
-						case ch <- path.WithImage(image):
-							return nil
-						}
+					}
+
+					select {
+					case <-ctx.Done():
+						return fmt.Errorf("canceled")
+					case ch <- path.WithImage(img):
+						return nil
 					}
 				})
 			}
@@ -206,7 +207,34 @@ func pathsToImages(
 	return ch, eg
 }
 
-func getImage(client *http.Client, ctx context.Context, url string) (image.Image, error) {
+func getImageWithPolicy(client *http.Client, ctx context.Context, path md.Path, policy DataSaverPolicy) (image.Image, error) {
+	resp := new(http.Response)
+	err := error(nil)
+
+	switch policy {
+	case DataSaverPolicyNo, DataSaverPolicyFallback:
+		resp, err = getResp(httpClient, ctx, path.DataURL)
+	case DataSaverPolicyPrefer:
+		resp, err = getResp(httpClient, ctx, path.DataSaverURL)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("download: %w", err)
+	}
+
+	img, _, err := image.Decode(resp.Body)
+	defer resp.Body.Close()
+
+	if err != nil && policy == DataSaverPolicyFallback {
+		return getImageWithPolicy(client, ctx, path, DataSaverPolicyPrefer)
+	} else if err != nil {
+		return nil, fmt.Errorf("decode: %w", err)
+	} else {
+		return img, nil
+	}
+}
+
+func getResp(client *http.Client, ctx context.Context, url string) (*http.Response, error) {
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, fmt.Errorf("prepare: %w", err)
@@ -215,17 +243,12 @@ func getImage(client *http.Client, ctx context.Context, url string) (image.Image
 	if err != nil {
 		return nil, fmt.Errorf("do: %w", err)
 	}
-	defer resp.Body.Close()
 
 	if resp.StatusCode != 200 {
 		return nil, fmt.Errorf("status: %v", resp.Status)
 	}
 
-	img, _, err := image.Decode(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("decode: %w", err)
-	}
-	return img, nil
+	return resp, nil
 }
 
 func bodyReadableErrorPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) {

cmd/root.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"runtime/pprof"
 
+	"github.com/leotaku/kojirou/cmd/formats/download"
 	"github.com/spf13/cobra"
 )
 
@@ -18,6 +19,7 @@ var (
 	forceArg            bool
 	leftToRightArg      bool
 	fillVolumeNumberArg int
+	dataSaverArg        download.DataSaverPolicy
 	diskArg             string
 	cpuprofileArg       string
 	groupsFilter        string
@@ -156,6 +158,7 @@ func init() {
 	rootCmd.Flags().BoolVarP(&kindleFolderModeArg, "kindle-folder-mode", "k", false, "generate folder structure for Kindle devices")
 	rootCmd.Flags().BoolVarP(&leftToRightArg, "left-to-right", "p", false, "make reading direction left to right")
 	rootCmd.Flags().IntVarP(&fillVolumeNumberArg, "fill-volume-number", "n", 0, "fill volume number with leading zeros in title")
+	rootCmd.Flags().VarP(&dataSaverArg, "data-saver", "s", "download lower quality images to save space")
 	rootCmd.Flags().BoolVarP(&dryRunArg, "dry-run", "d", false, "disable writing of any files")
 	rootCmd.Flags().StringVarP(&outArg, "out", "o", "", "output directory")
 	rootCmd.Flags().BoolVarP(&forceArg, "force", "f", false, "overwrite existing volumes")

mangadex/client.go

@@ -130,6 +130,8 @@ func (c *Client) FetchPaths(ctx context.Context, chapter *Chapter) (PathList, er
 	ah, err := c.base.GetAtHome(ctx, chapter.Info.ID)
 	if err != nil {
 		return nil, fmt.Errorf("get at home: %w", err)
+	} else if len(ah.Chapter.Data) != len(ah.Chapter.DataSaver) {
+		return nil, fmt.Errorf("broken chapter image list")
 	}
 
 	return convertChapter(chapter, ah), nil

mangadex/convert.go

@@ -61,7 +61,7 @@ func convertCovers(coverBaseURL string, mangaID string, co []api.CoverData) Path
 	for _, info := range co {
 		url := strings.Join([]string{coverBaseURL, mangaID, info.Attributes.FileName}, "/")
 		result = append(result, Path{
-			URL:               url,
+			DataURL:           url,
 			ImageIdentifier:   0,
 			ChapterIdentifier: NewIdentifier("0"),
 			VolumeIdentifier:  NewWithFallback(info.Attributes.Volume, "Special"),
@@ -73,10 +73,12 @@ func convertCovers(coverBaseURL string, mangaID string, co []api.CoverData) Path
 
 func convertChapter(ch *Chapter, ah *api.AtHome) PathList {
 	result := make(PathList, 0)
-	for i, filename := range ah.Chapter.Data {
-		url := strings.Join([]string{ah.BaseURL, "data", ah.Chapter.Hash, filename}, "/")
+	for i := range ah.Chapter.Data {
+		dataURL := strings.Join([]string{ah.BaseURL, "data", ah.Chapter.Hash, ah.Chapter.Data[i]}, "/")
+		dataSaverURL := strings.Join([]string{ah.BaseURL, "data-saver", ah.Chapter.Hash, ah.Chapter.DataSaver[i]}, "/")
 		result = append(result, Path{
-			URL:               url,
+			DataURL:           dataURL,
+			DataSaverURL:      dataSaverURL,
 			ImageIdentifier:   i,
 			ChapterIdentifier: ch.Info.Identifier,
 			VolumeIdentifier:  ch.Info.VolumeIdentifier,

mangadex/unstructured.go

@@ -41,7 +41,8 @@ type Image struct {
 }
 
 type Path struct {
-	URL string
+	DataURL      string
+	DataSaverURL string
 
 	// identifiers
 	ImageIdentifier   int