feat: add checksum validation

`bldr validate --checksums`. Refs #64. Signed-off-by: Alexey Palazhchenko <[email protected]>
siderolabs · Jul 14, 2021 · aa62d4a · aa62d4a
1 parent 07cd6ea
commit aa62d4a
Show file tree

Hide file tree

Showing 4 changed files with 272 additions and 68 deletions.
diff --git a/cmd/update.go b/cmd/update.go
@@ -30,6 +30,87 @@ type updateInfo struct {
 	*update.LatestInfo
 }
 
+//nolint:gocyclo
+func checkUpdates(ctx context.Context, set solver.PackageSet, l *log.Logger) error {
+	var (
+		wg          sync.WaitGroup
+		concurrency = runtime.GOMAXPROCS(-1)
+		sources     = make(chan *pkgInfo)
+		updates     = make(chan *updateInfo)
+	)
+
+	// start updaters
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+
+			for src := range sources {
+				res, e := update.Latest(ctx, src.source)
+				if e != nil {
+					l.Print(e)
+					continue
+				}
+
+				updates <- &updateInfo{
+					file:       src.file,
+					LatestInfo: res,
+				}
+			}
+		}()
+	}
+
+	var (
+		res  []updateInfo
+		done = make(chan struct{})
+	)
+
+	// start results reader
+	go func() {
+		for update := range updates {
+			res = append(res, *update)
+		}
+
+		close(done)
+	}()
+
+	// send work to updaters
+	for _, node := range set {
+		for _, step := range node.Pkg.Steps {
+			for _, src := range step.Sources {
+				sources <- &pkgInfo{
+					file:   node.Pkg.FileName,
+					source: src.URL,
+				}
+			}
+		}
+	}
+
+	close(sources)
+	wg.Wait()
+	close(updates)
+	<-done
+
+	sort.Slice(res, func(i, j int) bool { return res[i].file < res[j].file })
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
+	fmt.Fprintf(w, "%s\t%s\t%s\n", "File", "Update", "URL")
+
+	for _, info := range res {
+		if updateCmdFlag.all || info.HasUpdate {
+			url := info.LatestURL
+			if url == "" {
+				url = info.BaseURL
+			}
+
+			fmt.Fprintf(w, "%s\t%t\t%s\n", info.file, info.HasUpdate, url)
+		}
+	}
+
+	return w.Flush()
+}
+
 var updateCmdFlag struct {
 	all bool
 	dry bool
@@ -59,71 +140,7 @@ var updateCmd = &cobra.Command{
 			l.SetOutput(ioutil.Discard)
 		}
 
-		concurrency := runtime.GOMAXPROCS(-1)
-		var wg sync.WaitGroup
-		sources := make(chan *pkgInfo)
-		updates := make(chan *updateInfo)
-		for i := 0; i < concurrency; i++ {
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-
-				for src := range sources {
-					res, e := update.Latest(context.TODO(), src.source)
-					if e != nil {
-						l.Print(e)
-						continue
-					}
-
-					updates <- &updateInfo{
-						file:       src.file,
-						LatestInfo: res,
-					}
-				}
-			}()
-		}
-
-		var res []updateInfo
-		done := make(chan struct{})
-		go func() {
-			for update := range updates {
-				res = append(res, *update)
-			}
-			close(done)
-		}()
-
-		for _, node := range packages.ToSet() {
-			for _, step := range node.Pkg.Steps {
-				for _, src := range step.Sources {
-					sources <- &pkgInfo{
-						file:   node.Pkg.FileName,
-						source: src.URL,
-					}
-				}
-			}
-		}
-		close(sources)
-		wg.Wait()
-		close(updates)
-		<-done
-
-		sort.Slice(res, func(i, j int) bool { return res[i].file < res[j].file })
-
-		w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0)
-		fmt.Fprintf(w, "%s\t%s\t%s\n", "File", "Update", "URL")
-
-		for _, info := range res {
-			if updateCmdFlag.all || info.HasUpdate {
-				url := info.LatestURL
-				if url == "" {
-					url = info.BaseURL
-				}
-
-				fmt.Fprintf(w, "%s\t%t\t%s\n", info.file, info.HasUpdate, url)
-			}
-		}
-
-		if err = w.Flush(); err != nil {
+		if err = checkUpdates(context.TODO(), packages.ToSet(), l); err != nil {
 			log.Fatal(err)
 		}
 	},

diff --git a/cmd/validate.go b/cmd/validate.go
@@ -5,12 +5,81 @@
 package cmd
 
 import (
+	"context"
+	"fmt"
+	"io/ioutil"
 	"log"
+	"runtime"
+	"sync"
 
+	"github.com/hashicorp/go-multierror"
 	"github.com/spf13/cobra"
+
 	"github.com/talos-systems/bldr/internal/pkg/solver"
+	"github.com/talos-systems/bldr/internal/pkg/types/v1alpha2"
 )
 
+func validateChecksums(ctx context.Context, set solver.PackageSet, l *log.Logger) error {
+	var (
+		wg          sync.WaitGroup
+		concurrency = runtime.GOMAXPROCS(-1)
+		pkgs        = make(chan *v1alpha2.Pkg)
+		errors      = make(chan error)
+	)
+
+	// start downloaders
+	for i := 0; i < concurrency; i++ {
+		wg.Add(1)
+
+		go func() {
+			defer wg.Done()
+
+			for pkg := range pkgs {
+				for _, step := range pkg.Steps {
+					for _, src := range step.Sources {
+						l.Printf("downloading %s ...", src.URL)
+
+						_, _, err := src.ValidateChecksums(ctx)
+						if err != nil {
+							errors <- fmt.Errorf("%s: %w", pkg.Name, err)
+						}
+					}
+				}
+			}
+		}()
+	}
+
+	var (
+		multiErr *multierror.Error
+		done     = make(chan struct{})
+	)
+
+	// start results reader
+	go func() {
+		for err := range errors {
+			multiErr = multierror.Append(multiErr, err)
+		}
+
+		close(done)
+	}()
+
+	// send work to downloaders
+	for _, node := range set {
+		pkgs <- node.Pkg
+	}
+
+	close(pkgs)
+	wg.Wait()
+	close(errors)
+	<-done
+
+	return multiErr.ErrorOrNil()
+}
+
+var validateCmdFlags struct {
+	checksums bool
+}
+
 // validateCmd represents the validate command.
 var validateCmd = &cobra.Command{
 	Use:   "validate",
@@ -23,13 +92,25 @@ loads them and validates for errors. `,
 			Context: options.GetVariables(),
 		}
 
-		_, err := solver.NewPackages(&loader)
+		packages, err := solver.NewPackages(&loader)
 		if err != nil {
 			log.Fatal(err)
 		}
+
+		if validateCmdFlags.checksums {
+			l := log.New(log.Writer(), "[validate] ", log.Flags())
+			if !debug {
+				l.SetOutput(ioutil.Discard)
+			}
+
+			if err = validateChecksums(context.TODO(), packages.ToSet(), l); err != nil {
+				log.Fatal(err)
+			}
+		}
 	},
 }
 
 func init() {
+	validateCmd.Flags().BoolVar(&validateCmdFlags.checksums, "checksums", true, "validate checksums")
 	rootCmd.AddCommand(validateCmd)
 }
diff --git a/internal/pkg/types/v1alpha2/source.go b/internal/pkg/types/v1alpha2/source.go
@@ -5,8 +5,14 @@
 package v1alpha2
 
 import (
+	"context"
+	"crypto/sha256"
+	"crypto/sha512"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
 	"net/url"
 
 	"github.com/hashicorp/go-multierror"
@@ -53,13 +59,61 @@ func (source *Source) Validate() error {
 		multiErr = multierror.Append(multiErr, errors.New("source.destination can't be empty"))
 	}
 
-	if source.SHA256 == "" {
+	switch len(source.SHA256) {
+	case 0:
 		multiErr = multierror.Append(multiErr, errors.New("source.sha256 can't be empty"))
+	case 64: //nolint:gomnd
+		// nothing
+	default:
+		multiErr = multierror.Append(multiErr, errors.New("source.sha256 should be 64 chars long"))
 	}
 
-	if source.SHA512 == "" {
+	switch len(source.SHA512) {
+	case 0:
 		multiErr = multierror.Append(multiErr, errors.New("source.sha512 can't be empty"))
+	case 128: //nolint:gomnd
+		// nothing
+	default:
+		multiErr = multierror.Append(multiErr, errors.New("source.sha512 should be 128 chars long"))
 	}
 
 	return multiErr.ErrorOrNil()
 }
+
+// ValidateChecksums downloads the source, validates checksums,
+// and returns actual checksums and validation error, if any.
+func (source *Source) ValidateChecksums(ctx context.Context) (string, string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", source.URL, nil)
+	if err != nil {
+		return "", "", err
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+
+	defer resp.Body.Close() //nolint:errcheck
+
+	s256 := sha256.New()
+	s512 := sha512.New()
+
+	if _, err = io.Copy(io.MultiWriter(s256, s512), resp.Body); err != nil {
+		return "", "", err
+	}
+
+	var (
+		actualSHA256, actualSHA512 string
+		multiErr                   *multierror.Error
+	)
+
+	if actualSHA256 = hex.EncodeToString(s256.Sum(nil)); source.SHA256 != actualSHA256 {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("source.sha256 does not match: expected %s, got %s", source.SHA256, actualSHA256))
+	}
+
+	if actualSHA512 = hex.EncodeToString(s512.Sum(nil)); source.SHA512 != actualSHA512 {
+		multiErr = multierror.Append(multiErr, fmt.Errorf("source.sha512 does not match: expected %s, got %s", source.SHA512, actualSHA512))
+	}
+
+	return actualSHA256, actualSHA512, multiErr.ErrorOrNil()
+}