nydusify: add chunkdict generate command and corresponding tests

Signed-off-by: Lin Wang <[email protected]>

contrib/nydusify/cmd/nydusify.go

@@ -657,12 +657,45 @@ func main() {
 							Usage:    "One or more Nydus image reference(Multiple images should be split by commas)",
 							EnvVars:  []string{"SOURCES"},
 						},
+						&cli.StringFlag{
+							Name:     "target",
+							Required: false,
+							Usage:    "Target chunkdict image (Nydus) reference",
+							EnvVars:  []string{"TARGET"},
+						},
 						&cli.BoolFlag{
 							Name:     "source-insecure",
 							Required: false,
 							Usage:    "Skip verifying server certs for HTTPS source registry",
 							EnvVars:  []string{"SOURCE_INSECURE"},
 						},
+						&cli.BoolFlag{
+							Name:     "target-insecure",
+							Required: false,
+							Usage:    "Skip verifying server certs for HTTPS target registry",
+							EnvVars:  []string{"TARGET_INSECURE"},
+						},
+
+						&cli.StringFlag{
+							Name:    "backend-type",
+							Value:   "",
+							Usage:   "Type of storage backend, possible values: 'oss', 's3'",
+							EnvVars: []string{"BACKEND_TYPE"},
+						},
+						&cli.StringFlag{
+							Name:    "backend-config",
+							Value:   "",
+							Usage:   "Json configuration string for storage backend",
+							EnvVars: []string{"BACKEND_CONFIG"},
+						},
+						&cli.PathFlag{
+							Name:      "backend-config-file",
+							Value:     "",
+							TakesFile: true,
+							Usage:     "Json configuration file for storage backend",
+							EnvVars:   []string{"BACKEND_CONFIG_FILE"},
+						},
+
 						&cli.StringFlag{
 							Name:    "work-dir",
 							Value:   "./output",
@@ -675,6 +708,12 @@ func main() {
 							Usage:   "Path to the nydus-image binary, default to search in PATH",
 							EnvVars: []string{"NYDUS_IMAGE"},
 						},
+
+						&cli.BoolFlag{
+							Name:  "all-platforms",
+							Value: false,
+							Usage: "Generate chunkdict image for all platforms, conflicts with --platform",
+						},
 						&cli.StringFlag{
 							Name:  "platform",
 							Value: "linux/" + runtime.GOARCH,
@@ -684,17 +723,31 @@ func main() {
 					Action: func(c *cli.Context) error {
 						setupLogLevel(c)
 
+						backendType, backendConfig, err := getBackendConfig(c, "", false)
+						if err != nil {
+							return err
+						}
+
 						_, arch, err := provider.ExtractOsArch(c.String("platform"))
 						if err != nil {
 							return err
 						}
 
 						generator, err := generator.New(generator.Opt{
-							WorkDir:        c.String("work-dir"),
 							Sources:        c.StringSlice("sources"),
+							Target:         c.String("target"),
 							SourceInsecure: c.Bool("source-insecure"),
+							TargetInsecure: c.Bool("target-insecure"),
+
+							BackendType:      backendType,
+							BackendConfig:    backendConfig,
+							BackendForcePush: c.Bool("backend-force-push"),
+
+							WorkDir:        c.String("work-dir"),
 							NydusImagePath: c.String("nydus-image"),
 							ExpectedArch:   arch,
+							AllPlatforms:   c.Bool("all-platforms"),
+							Platforms:      c.String("platform"),
 						})
 						if err != nil {
 							return err

contrib/nydusify/pkg/build/builder.go

@@ -41,8 +41,11 @@ type CompactOption struct {
 	CompactConfigPath   string
 }
 
-type SaveOption struct {
-	BootstrapPath string
+type GenerateOption struct {
+	BootstrapPaths         []string
+	DatabasePath           string
+	ChunkdictBootstrapPath string
+	OutputPath             string
 }
 
 type Builder struct {
@@ -148,15 +151,22 @@ func (builder *Builder) Run(option BuilderOption) error {
 	return builder.run(args, option.PrefetchPatterns)
 }
 
-// Save calls `nydus-image chunkdict save` to parse Nydus bootstrap
-func (builder *Builder) Save(option SaveOption) error {
+// Generate calls `nydus-image chunkdict generate` to get chunkdict
+func (builder *Builder) Generate(option GenerateOption) error {
+	logrus.Infof("Invoking 'nydus-image chunkdict generate' command")
 	args := []string{
 		"chunkdict",
-		"save",
+		"generate",
 		"--log-level",
 		"warn",
 		"--bootstrap",
-		option.BootstrapPath,
+		option.ChunkdictBootstrapPath,
+		"--database",
+		option.DatabasePath,
+		"--output-json",
+		option.OutputPath,
 	}
+	args = append(args, option.BootstrapPaths...)
+
 	return builder.run(args, "")
 }