diff --git a/cmd/cmd_pkg.go b/cmd/cmd_pkg.go index 0c08dc7e..cd8a7267 100644 --- a/cmd/cmd_pkg.go +++ b/cmd/cmd_pkg.go @@ -114,18 +114,23 @@ func PackageCommands() *cobra.Command { } func cmdListPackages(cmd *cobra.Command, args []string) { - var packages *map[string]api.Package + var packages []api.Package var err error local, _ := cmd.Flags().GetBool("local") if local { packages, err = api.GetLocalPackageList() + if err != nil { + log.Errorf("failed getting packages: %s", err) + return + } } else { - packages, err = api.GetPackageList(api.NewConfig()) - } - if err != nil { - log.Errorf("failed getting packages: %s", err) - return + pkgList, err := api.GetPackageList(api.NewConfig()) + if err != nil { + log.Errorf("failed getting packages: %s", err) + return + } + packages = pkgList.List() } searchRegex, err := cmd.Flags().GetString("search") @@ -134,12 +139,13 @@ func cmdListPackages(cmd *cobra.Command, args []string) { } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"PackageName", "Version", "Language", "Runtime", "Description"}) + table.SetHeader([]string{"Namespace", "PackageName", "Version", "Language", "Runtime", "Description"}) table.SetHeaderColor( tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}, tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}, tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}, tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}, + tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}, tablewriter.Colors{tablewriter.Bold, tablewriter.FgCyanColor}) table.SetRowLine(true) @@ -156,31 +162,37 @@ func cmdListPackages(cmd *cobra.Command, args []string) { } // Sort the package list by packagename - keys := make([]string, 0, len(*packages)) - for key := range *packages { - keys = append(keys, key) + keys := make([]string, 0, len(packages)) + for _, pkg := range packages { + keys = append(keys, pkg.Name) } sort.Slice(keys, func(i, j int) bool { return strings.ToLower(keys[i]) < strings.ToLower(keys[j]) }) - for _, key := range keys { + var rows [][]string + for _, pkg := range packages { var row []string // If we are told to filter and get no matches then filter out the // current row. If we are not told to filter then just add the // row. if filter && - !(r.MatchString((*packages)[key].Language) || - r.MatchString((*packages)[key].Runtime) || - r.MatchString(key)) { + !(r.MatchString(pkg.Language) || + r.MatchString(pkg.Runtime) || + r.MatchString(pkg.Name) || + r.MatchString(pkg.Namespace)) { continue } + row = append(row, pkg.Namespace) + row = append(row, pkg.Name) + row = append(row, pkg.Version) + row = append(row, pkg.Language) + row = append(row, pkg.Runtime) + row = append(row, pkg.Description) + rows = append(rows, row) + } - row = append(row, key) - row = append(row, (*packages)[key].Version) - row = append(row, (*packages)[key].Language) - row = append(row, (*packages)[key].Runtime) - row = append(row, (*packages)[key].Description) + for _, row := range rows { table.Append(row) } @@ -188,10 +200,20 @@ func cmdListPackages(cmd *cobra.Command, args []string) { } func cmdGetPackage(cmd *cobra.Command, args []string) { + identifier := args[0] + tokens := strings.Split(identifier, "/") + if len(tokens) < 2 { + log.Fatal(errors.New("invalid package name. expected format /:")) + } downloadPackage(args[0], api.NewConfig()) } func cmdPackageDescribe(cmd *cobra.Command, args []string) { + identifier := args[0] + tokens := strings.Split(identifier, "/") + if len(tokens) < 2 { + log.Fatal(errors.New("invalid package name. expected format /:")) + } expackage := filepath.Join(packageDirectoryPath(), args[0]) if _, err := os.Stat(expackage); os.IsNotExist(err) { expackage = downloadPackage(args[0], api.NewConfig()) @@ -222,6 +244,11 @@ func cmdPackageDescribe(cmd *cobra.Command, args []string) { func cmdPackageContents(cmd *cobra.Command, args []string) { flags := cmd.Flags() + identifier := args[0] + tokens := strings.Split(identifier, "/") + if len(tokens) < 2 { + log.Fatal(errors.New("invalid package name. expected format /:")) + } directoryPath := packageDirectoryPath() @@ -229,7 +256,7 @@ func cmdPackageContents(cmd *cobra.Command, args []string) { directoryPath = localPackageDirectoryPath() } - expackage := filepath.Join(directoryPath, args[0]) + expackage := filepath.Join(directoryPath, strings.ReplaceAll(args[0], ":", "_")) if _, err := os.Stat(expackage); os.IsNotExist(err) { expackage = downloadPackage(args[0], api.NewConfig()) } @@ -318,8 +345,8 @@ func cmdPkgPush(cmd *cobra.Command, args []string) { } localPackages := filepath.Join(api.GetOpsHome(), "local_packages") var foundPkg api.Package - for name, pkg := range *pkgList { - if name == packageName { + for _, pkg := range pkgList { + if pkg.Name == packageName { foundPkg = pkg break } @@ -396,6 +423,7 @@ func loadCommandHandler(cmd *cobra.Command, args []string) { runLocalInstanceFlags := NewRunLocalInstanceCommandFlags(flags) pkgFlags := NewPkgCommandFlags(flags) pkgFlags.Package = args[0] + pkgFlags.SluggedPackage = strings.ReplaceAll(args[0], ":", "_") c := api.NewConfig() diff --git a/cmd/cmd_pkg_test.go b/cmd/cmd_pkg_test.go index 21b3fbdc..312b9d3e 100644 --- a/cmd/cmd_pkg_test.go +++ b/cmd/cmd_pkg_test.go @@ -21,7 +21,7 @@ func TestListPkgCommand(t *testing.T) { func TestGetPkgCommand(t *testing.T) { getPkgCmd := cmd.PackageCommands() - getPkgCmd.SetArgs([]string{"get", "node_v14.2.0"}) + getPkgCmd.SetArgs([]string{"get", "eyberg/bind:9.13.4"}) err := getPkgCmd.Execute() @@ -31,7 +31,7 @@ func TestGetPkgCommand(t *testing.T) { func TestPkgContentsCommand(t *testing.T) { getPkgCmd := cmd.PackageCommands() - getPkgCmd.SetArgs([]string{"contents", "node_v14.2.0"}) + getPkgCmd.SetArgs([]string{"contents", "eyberg/bind:9.13.4"}) err := getPkgCmd.Execute() @@ -41,7 +41,7 @@ func TestPkgContentsCommand(t *testing.T) { func TestPkgDescribeCommand(t *testing.T) { getPkgCmd := cmd.PackageCommands() - getPkgCmd.SetArgs([]string{"describe", "node_v14.2.0"}) + getPkgCmd.SetArgs([]string{"describe", "eyberg/bind:9.13.4"}) err := getPkgCmd.Execute() @@ -54,7 +54,7 @@ func TestLoad(t *testing.T) { program := buildNodejsProgram() defer os.Remove(program) - getPkgCmd.SetArgs([]string{"load", "node_v14.2.0", "-a", program}) + getPkgCmd.SetArgs([]string{"load", "eyberg/node:v14.2.0", "-a", program}) err := getPkgCmd.Execute() diff --git a/cmd/download_package.go b/cmd/download_package.go index 37bdb692..3339543d 100644 --- a/cmd/download_package.go +++ b/cmd/download_package.go @@ -154,13 +154,13 @@ func downloadAndExtractPackage(packagesDirPath, pkg string, config *types.Config log.Fatal(err) } - expackage := path.Join(packagesDirPath, pkg) + expackage := path.Join(packagesDirPath, strings.ReplaceAll(pkg, ":", "_")) opsPackage, err := api.DownloadPackage(pkg, config) if err != nil { log.Fatal(err) } - api.ExtractPackage(opsPackage, packagesDirPath, config) + api.ExtractPackage(opsPackage, path.Dir(expackage), config) err = os.Remove(opsPackage) if err != nil { diff --git a/cmd/flags_pkg.go b/cmd/flags_pkg.go index 44c987c0..418350c8 100644 --- a/cmd/flags_pkg.go +++ b/cmd/flags_pkg.go @@ -16,8 +16,9 @@ import ( // PkgCommandFlags consolidates all command flags required to use a provider type PkgCommandFlags struct { - Package string - LocalPackage bool + Package string + SluggedPackage string + LocalPackage bool } // PackagePath returns the package path in file system @@ -26,7 +27,7 @@ func (flags *PkgCommandFlags) PackagePath() string { return path.Join(api.GetOpsHome(), "local_packages", flags.Package) } - return path.Join(api.GetOpsHome(), "packages", flags.Package) + return path.Join(api.GetOpsHome(), "packages", flags.SluggedPackage) } // MergeToConfig merge package configuration to ops configuration diff --git a/lepton/const.go b/lepton/const.go index c2f32340..48a3fcc2 100644 --- a/lepton/const.go +++ b/lepton/const.go @@ -26,10 +26,10 @@ const releaseBaseURL string = "https://storage.googleapis.com/nanos/release/" const nightlyReleaseBaseURL string = "https://storage.googleapis.com/nanos/release/nightly/" // PackageBaseURL gives URL for downloading of packages -const PackageBaseURL string = PkghubBaseURL + "/packages" +const PackageBaseURL string = PkghubBaseURL + "/v2/packages" // PackageManifestURL stores info about all packages -const PackageManifestURL string = PkghubBaseURL + "/manifest.json" +const PackageManifestURL string = PkghubBaseURL + "/v2/manifest.json" // PackageManifestFileName is manifest file path const PackageManifestFileName string = "manifest.json" diff --git a/lepton/image.go b/lepton/image.go index 0fa9b87f..952a8ac3 100644 --- a/lepton/image.go +++ b/lepton/image.go @@ -561,7 +561,7 @@ func DownloadFileWithProgress(filepath string, url string, timeout int) error { // DownloadFile downloads file using URL func DownloadFile(filepath string, url string, timeout int, showProgress bool) error { - log.Info("Downloading..", url) + log.Info("Downloading..", url, "to", filepath) out, err := os.Create(filepath + ".tmp") if err != nil { return err diff --git a/lepton/package.go b/lepton/package.go index af26b8d4..d001c290 100644 --- a/lepton/package.go +++ b/lepton/package.go @@ -27,7 +27,8 @@ const PackageSysRootFolderName = "sysroot" // PackageList contains a list of known packages. type PackageList struct { - list map[string]Package + Version int `json:"Version"` + Packages []Package `json:"Packages"` } // Package is the definition of an OPS package. @@ -37,20 +38,72 @@ type Package struct { Language string `json:"language"` Description string `json:"description,omitempty"` SHA256 string `json:"sha256"` + Name string `json:"name"` + Namespace string `json:"namespace"` } -// DownloadPackage downloads package by name -func DownloadPackage(name string, config *types.Config) (string, error) { +// PackageIdentifier is used to identify a namespaced package +type PackageIdentifier struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Version string `json:"version"` +} + +// Match matches a package with all the fields of this identifier +func (pkgidf *PackageIdentifier) Match(pkg Package) bool { + return pkg.Name == pkgidf.Name && pkg.Namespace == pkgidf.Namespace && pkg.Version == pkgidf.Version +} + +// FindPackage finds a package with the mentioned identifier +func (pkglist *PackageList) FindPackage(identifier string) (*Package, bool) { + idf := ParseIdentifier(identifier) + for _, pkg := range pkglist.Packages { + if idf.Match(pkg) { + return &pkg, true + } + } + return nil, false +} + +// List returns the package list +func (pkglist *PackageList) List() []Package { + return pkglist.Packages +} + +// ParseIdentifier parses a package identifier which looks like /: +func ParseIdentifier(identifier string) PackageIdentifier { + tokens := strings.Split(identifier, "/") + namespace := tokens[len(tokens)-2] + pkgTokens := strings.Split(tokens[len(tokens)-1], ":") + pkgName := pkgTokens[0] + version := "latest" + if len(pkgTokens) > 1 { + version = pkgTokens[1] + } + return PackageIdentifier{ + Name: pkgName, + Namespace: namespace, + Version: version, + } +} + +// DownloadPackage downloads package by identifier +func DownloadPackage(identifier string, config *types.Config) (string, error) { packages, err := GetPackageList(config) if err != nil { return "", nil } - if _, ok := (*packages)[name]; !ok { - return "", fmt.Errorf("package %q does not exist", name) + pkg, ok := packages.FindPackage(identifier) + + if !ok { + return "", fmt.Errorf("package %q does not exist", identifier) } - archivename := name + ".tar.gz" + archivename := pkg.Namespace + "/" + pkg.Name + "_" + pkg.Version + ".tar.gz" + + archiveFolder := path.Join(PackagesCache, pkg.Namespace) + os.MkdirAll(archiveFolder, 0755) packagepath := path.Join(PackagesCache, archivename) _, err = os.Stat(packagepath) if err != nil && !os.IsNotExist(err) { @@ -61,6 +114,8 @@ func DownloadPackage(name string, config *types.Config) (string, error) { return packagepath, nil } + archivePath := pkg.Namespace + "/" + pkg.Name + "/" + pkg.Version + ".tar.gz" + pkgBaseURL := PackageBaseURL // Check config override @@ -81,9 +136,9 @@ func DownloadPackage(name string, config *types.Config) (string, error) { if isNetworkRepo { var fileURL string if strings.HasSuffix(pkgBaseURL, "/") { - fileURL = pkgBaseURL + archivename + fileURL = pkgBaseURL + archivePath } else { - fileURL = fmt.Sprintf("%s/%s", pkgBaseURL, archivename) + fileURL = fmt.Sprintf("%s/%s", pkgBaseURL, archivePath) } if err = DownloadFileWithProgress(packagepath, fileURL, 600); err != nil { @@ -125,7 +180,7 @@ func DownloadPackage(name string, config *types.Config) (string, error) { } // GetPackageList provides list of packages -func GetPackageList(config *types.Config) (*map[string]Package, error) { +func GetPackageList(config *types.Config) (*PackageList, error) { var err error pkgManifestURL := PackageManifestURL @@ -177,17 +232,17 @@ func GetPackageList(config *types.Config) (*map[string]Package, error) { return nil, err } - err = json.Unmarshal(data, &packages.list) + err = json.Unmarshal(data, &packages) if err != nil { return nil, err } - return &packages.list, nil + return &packages, nil } // GetLocalPackageList provides list of local packages -func GetLocalPackageList() (*map[string]Package, error) { - packages := map[string]Package{} +func GetLocalPackageList() ([]Package, error) { + packages := []Package{} localPackagesDir := GetOpsHome() + "/local_packages" @@ -214,11 +269,11 @@ func GetLocalPackageList() (*map[string]Package, error) { return nil, err } - packages[pkgName] = pkg + packages = append(packages, pkg) } } - return &packages, nil + return packages, nil } func getPackageCache() string { @@ -271,21 +326,23 @@ func ExtractPackage(archive, dest string, config *types.Config) { sha := sha256Of(archive) homeDirName := filepath.Base(GetOpsHome()) + pkgList, err := GetPackageList(config) + if err != nil { + panic(err) + } // hack // this only verifies for packages - unfortunately this function is // used for extracting releases (which currently don't have // checksums) if strings.Contains(archive, filepath.Join(homeDirName, "packages")) { - fname := filepath.Base(archive) + namespace := filepath.Base(filepath.Dir(archive)) fname = strings.ReplaceAll(fname, ".tar.gz", "") - - list, err := GetPackageList(config) - if err != nil { - panic(err) - } - - if (*list)[fname].SHA256 != sha { + fnameTokens := strings.Split(fname, "_") + pkgName := fnameTokens[0] + version := fnameTokens[len(fnameTokens)-1] + pkg, found := pkgList.FindPackage(fmt.Sprintf("%s/%s:%s", namespace, pkgName, version)) + if !found || pkg == nil || pkg.SHA256 != sha { log.Fatalf("This package doesn't match what is in the manifest.") }