Initial resolver working

Signed-off-by: Gavin Zhao <[email protected]>

builder/build.go

@@ -229,12 +229,9 @@ func (p *Package) CopyAssets(h *PackageHistory, o *Overlay) error {
 	return h.WriteXML(histPath)
 }
 
-func (p *Package) calcDeps(profile *Profile) (deps []string) {
+func (p *Package) calcDeps(resolver *Resolver) ([]Dep, error) {
 	// hash = LayersFakeHash
-	deps = append(deps, "rust")
-	deps = append(deps, "cargo")
-	deps = append(deps, "llvm")
-	return
+	return resolver.Query(p.Deps, true, true)
 }
 
 // PrepYpkg will do the initial leg work of preparing us for a ypkg build.
@@ -491,7 +488,7 @@ func (p *Package) CollectAssets(overlay *Overlay, usr *UserInfo, manifestTarget
 }
 
 // Build will attempt to build the package in the overlayfs system.
-func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Profile, pman *EopkgManager, overlay *Overlay, manifestTarget string) error {
+func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Profile, pman *EopkgManager, overlay *Overlay, resolver *Resolver, manifestTarget string) error {
 	slog.Debug("Building package", "name", p.Name, "version", p.Version, "release", p.Release, "type", p.Type,
 		"profile", overlay.Back.Name)
 
@@ -508,7 +505,11 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro
 
 	// Set up layers caching, only for YPKG
 	if p.Type == PackageTypeYpkg {
-		deps := p.calcDeps(profile)
+		deps, err := p.calcDeps(resolver)
+		if err != nil {
+			return fmt.Errorf("Failed to calculate dependencies: %w", err)
+		}
+
 		layer := Layer{
 			deps:    deps,
 			profile: profile,
@@ -520,6 +521,7 @@ func (p *Package) Build(notif PidNotifier, history *PackageHistory, profile *Pro
 			return err
 		}
 		overlay.LayerDir = contentPath
+		slog.Info("Using layer", "hash", layer.Hash())
 	} else {
 		return errors.New("Under testing of layers feature, XML build is not enabled yet.")
 	}

builder/layer.go

@@ -5,23 +5,26 @@ import (
 	"encoding/json"
 	"fmt"
 	"log/slog"
+	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/zeebo/blake3"
 )
 
 type Layer struct {
-	deps    []string
+	deps    []Dep
 	profile *Profile
 	back    *BackingImage
+	created bool
+	hash    string
 }
 
 func (l Layer) MarshalJSON() ([]byte, error) {
 	var imageHash string
 	var err error
 	if PathExists(l.back.ImagePath) {
-		if imageHash, err = hashFile(l.back.ImagePath); err != nil {
+		if imageHash, err = xxh3128HashFile(l.back.ImagePath); err != nil {
 			return nil, err
 		}
 		// } else if PathExists(l.back.ImagePath) {
@@ -33,19 +36,23 @@ func (l Layer) MarshalJSON() ([]byte, error) {
 	}
 
 	return json.Marshal(struct {
-		Deps      []string
+		Deps      []Dep `json:"deps"`
 		ImageHash string
 	}{Deps: l.deps, ImageHash: imageHash})
 }
 
 func (l *Layer) Hash() string {
-	jsonBytes, err := json.Marshal(l)
-	if err != nil {
-		return LayersFakeHash
-	} else {
-		hashBytes := blake3.Sum256(jsonBytes)
-		return base64.StdEncoding.EncodeToString(hashBytes[:])
+	if l.hash == "" {
+		jsonBytes, err := json.Marshal(l)
+		if err != nil {
+			l.hash = LayersFakeHash
+		} else {
+			hashBytes := blake3.Sum256(jsonBytes)
+			l.hash = base64.StdEncoding.EncodeToString(hashBytes[:])
+		}
 	}
+	return l.hash
+
 }
 
 func (l *Layer) BasePath() string {
@@ -61,7 +68,15 @@ func (l *Layer) RequestOverlay(notif PidNotifier) (contentPath string, err error
 	}
 }
 
+func (l *Layer) RemoveIfNotCreated() {
+	slog.Debug("Layer not fully created, removing...", "path", l.BasePath())
+	if !l.created {
+		os.RemoveAll(l.BasePath())
+	}
+}
+
 func (l *Layer) Create(notif PidNotifier) (contentPath string, err error) {
+	slog.Info("Creating layer", "hash", l.Hash())
 	basePath := l.BasePath()
 	contentPath = filepath.Join(basePath, "content")
 
@@ -136,7 +151,12 @@ func (l *Layer) Create(notif PidNotifier) (contentPath string, err error) {
 	}
 
 	// Install our dependencies
-	cmd := fmt.Sprintf("eopkg it -y %s", strings.Join(l.deps, " "))
+	pkgs := make([]string, len(l.deps))
+	for idx, dep := range l.deps {
+		pkgs[idx] = dep.Name
+	}
+	slog.Debug("Installing dependencies", "size", len(pkgs), "pkgs", pkgs)
+	cmd := fmt.Sprintf("eopkg it -y %s", strings.Join(pkgs, " "))
 	if DisableColors {
 		cmd += " -n"
 	}

builder/manager.go

@@ -17,19 +17,25 @@
 package builder
 
 import (
+	"encoding/xml"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
+	"net/http"
 	"os"
 	"os/signal"
+	"path"
 	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"github.com/getsolus/libeopkg/index"
 	"github.com/getsolus/libosdev/disk"
 	"github.com/go-git/go-git/v5"
+	"github.com/ulikunitz/xz"
 
 	"github.com/getsolus/solbuild/cli/log"
 )
@@ -72,6 +78,7 @@ type Manager struct {
 	pkgManager *EopkgManager // Package manager, if any
 	lock       *sync.Mutex   // Lock on all operations to prevent.. damage.
 	profile    *Profile      // The profile we've been requested to use
+	resolver   *Resolver
 
 	lockfile *LockFile // We track the global lock for each operation
 	didStart bool      // Whether we got anything done.
@@ -374,7 +381,10 @@ func (m *Manager) Build() error {
 		return err
 	}
 
-	return m.pkg.Build(m, m.history, m.GetProfile(), m.pkgManager, m.overlay, m.manifestTarget)
+	m.InitResolver()
+	slog.Debug("Successfully initialized resolver")
+
+	return m.pkg.Build(m, m.history, m.GetProfile(), m.pkgManager, m.overlay, m.resolver, m.manifestTarget)
 }
 
 // Chroot will enter the build environment to allow users to introspect it.
@@ -478,3 +488,92 @@ func (m *Manager) SetTmpfs(enable bool, size string) {
 		m.Config.TmpfsSize = strings.TrimSpace(size)
 	}
 }
+
+func (m *Manager) InitResolver() error {
+	m.resolver = NewResolver()
+
+	if m.profile == nil {
+		return errors.New("Profile not initialized!")
+	}
+
+	profile := m.profile
+	/// nameToUrl := make(map[string]string)
+	repos := []string{}
+
+	if strings.Contains(profile.Image, "unstable") {
+		// nameToUrl["Solus"] = "https://cdn.getsol.us/repo/unstable/eopkg-index.xml.xz"
+		repos = append(repos, "https://cdn.getsol.us/repo/unstable/eopkg-index.xml.xz")
+		// repos = append(repos, "https://packages.getsol.us/unstable/eopkg-index.xml.xz")
+	} else if strings.Contains(profile.Image, "stable") {
+		// nameToUrl["Solus"] = "https://cdn.getsol.us/repo/shannon/eopkg-index.xml.xz"
+		repos = append(repos, "https://cdn.getsol.us/repo/shannon/eopkg-index.xml.xz")
+	} else {
+		slog.Warn("Unrecognized image name, not adding default repo", "image", profile.Image)
+	}
+
+	// Realistically, remove can only be * or Solus
+	// for _, remove := range profile.RemoveRepos {
+	// 	if remove == "*" {
+	// 		repos = []string{}
+	// 		continue
+	// 	}
+
+	// 	if idx := slices.Index(repos, remove); idx != -1 {
+	// 		repos = slices.Delete(repos, idx, idx+1)
+	// 	} else {
+	// 		slog.Warn("Cannot remove noexistent repo", "name", remove)
+	// 	}
+	// }
+	if len(profile.RemoveRepos) != 0 {
+		repos = []string{}
+		if len(profile.RemoveRepos) > 1 {
+			slog.Warn("Unexpectedly requested removing of more than 1 repo", "removes", profile.RemoveRepos)
+		}
+	}
+
+	for _, add := range profile.AddRepos {
+		if repo := profile.Repos[add]; repo != nil {
+			repos = append(repos, repo.URI)
+		} else {
+			slog.Warn("Cannot add nonexistent repo", "name", add)
+		}
+	}
+
+	for _, repo := range repos {
+		slog.Debug("Fetching repo", "url", repo)
+
+		var r io.Reader
+		ext := path.Ext(repo)
+		resp, err := http.Get(repo)
+		if err != nil {
+			// slog.Error("Failed to fetch", "url", repo, "error", err)
+			return fmt.Errorf("Failed to fetch %s: %w", repo, err)
+		}
+		slog.Debug("Fetched")
+
+		if ext == ".xz" {
+			// slog.Debug("Decoding .xz")
+			if r, err = xz.NewReader(resp.Body); err != nil {
+				// slog.Error("Failed to init xz reader", "error", err)
+				return fmt.Errorf("Failed to init xz reader for %s: %w", repo, err)
+			}
+		} else if ext == ".xml" {
+			r = resp.Body
+		} else {
+			// slog.Error("Unrecognized repo url extension", "url", repo, "ext", ext)
+			return fmt.Errorf("Unrecognized repo url extension %s for %s", ext, repo)
+		}
+
+		dec := xml.NewDecoder(r)
+		var i index.Index
+		if err := dec.Decode(&i); err != nil {
+			// slog.Error("Failed to decode index", "error", err)
+			return fmt.Errorf("Failed to decode index for %s: %w", repo, err)
+		}
+
+		m.resolver.AddIndex(&i)
+		slog.Info("Parsed and added repo to resolver", "url", repo)
+	}
+
+	return nil
+}

builder/pkg.go

@@ -62,6 +62,7 @@ type Package struct {
 	Path       string          // Path to the build spec
 	Sources    []source.Source // Each package has 0 or more sources that we fetch
 	CanNetwork bool            // Only applicable to ypkg builds
+	Deps       []string
 }
 
 // YmlPackage is a parsed ypkg build file.
@@ -71,6 +72,8 @@ type YmlPackage struct {
 	Release    int                 `yaml:"release"`
 	Networking bool                `yaml:"networking"` // If set to false (default) we disable networking in the build
 	Source     []map[string]string `yaml:"source"`
+	BuildDeps  []string              `yaml:"builddeps"`
+	CheckDeps  []string              `yaml:"checkdeps"`
 }
 
 // XMLUpdate represents an update in the package history.
@@ -220,6 +223,7 @@ func NewYmlPackageFromBytes(by []byte) (*Package, error) {
 		Release:    ypkg.Release,
 		Type:       PackageTypeYpkg,
 		CanNetwork: ypkg.Networking,
+		Deps: append(ypkg.BuildDeps, ypkg.CheckDeps...),
 	}
 
 	for _, row := range ypkg.Source {