From f0c4d73fc66447f808902cbba294d882400660a2 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Fri, 23 Aug 2024 12:23:06 -0600 Subject: [PATCH] Make all new ENVs opt in: The following envs were added as opt in: RETRY_ENABLED RETRY_DURATION_MINUTES PROGRESS_INTERVAL_SECONDS TEXT_LOGGING Signed-off-by: Jacob Weinstock --- image2disk/Dockerfile | 8 ++-- image2disk/README.md | 30 +++++++++--- image2disk/image/image.go | 16 +++---- image2disk/main.go | 96 +++++++++++++++++++++++++++++++------ rootio/storage/partition.go | 2 +- 5 files changed, 116 insertions(+), 36 deletions(-) diff --git a/image2disk/Dockerfile b/image2disk/Dockerfile index e705d16..f5559ec 100644 --- a/image2disk/Dockerfile +++ b/image2disk/Dockerfile @@ -1,10 +1,10 @@ FROM golang:1.23-alpine AS image2disk RUN apk add --no-cache git ca-certificates gcc linux-headers musl-dev -COPY . /src WORKDIR /src/image2disk -RUN --mount=type=cache,sharing=locked,id=gomod,target=/go/pkg/mod/cache \ - --mount=type=cache,sharing=locked,id=goroot,target=/root/.cache/go-build \ - CGO_ENABLED=1 GOOS=linux go build -a -ldflags "-linkmode external -extldflags '-static' -s -w" -o image2disk +COPY go.mod go.sum /src/ +RUN go mod download +COPY . /src +RUN CGO_ENABLED=1 GOOS=linux go build -a -ldflags "-linkmode external -extldflags '-static' -s -w" -o image2disk FROM scratch # Add Certificates into the image, for anything that does HTTPS calls diff --git a/image2disk/README.md b/image2disk/README.md index c49f9bf..671b6e4 100644 --- a/image2disk/README.md +++ b/image2disk/README.md @@ -1,15 +1,31 @@ -``` +# image2disk + +```bash quay.io/tinkerbell/actions/image2disk:latest ``` -This action will stream a remote disk image (raw) to a block device, and +This Action will stream a remote disk image (raw) to a block device, and is mainly used to write cloud images to a disk. It is recommended to use the `qemu-img` tool to convert disk images into raw, it is also possible to compress the raw images -with tar+gzip to prevent wasted disk space +with tar+gzip to prevent wasted disk space. + +| env var | data type | default value | required | description | +|---------|-----------|---------------|----------|-------------| +| IMG_URL | string | "" | yes | URL of the image to be streamed | +| DEST_DISK | string | "" | yes | Block device to which to write the image | +| COMPRESSED | bool | false | no | Decompress the image before writing it to the disk | +| RETRY_ENABLED | bool | false | no | Retry the Action, using exponential backoff, for the duration specified in `RETRY_DURATION_MINUTES` before failing | +| RETRY_DURATION_MINUTES | int | 10 | no | Duration for which the Action will retry before failing | +| PROGRESS_INTERVAL_SECONDS | int | 3 | no | Interval at which the progress of the image transfer will be logged | +| TEXT_LOGGING | bool | false | no | Output from the Action will be logged in a more human friendly text format, JSON format is used by default. | The below example will stream a raw ubuntu cloud image (converted by qemu-img) and write it to the block storage disk `/dev/sda`. The raw image is uncompressed in this example. +```bash +qemu-img convert ubuntu.img ubuntu.raw +``` + ```yaml actions: - name: "stream ubuntu" @@ -21,10 +37,10 @@ actions: COMPRESSED: false ``` -The below example will stream a compressed raw ubuntu cloud image (converted by qemu-img) -and then compressed with gzip to reduce local space. +The below example will stream a compressed raw ubuntu cloud image (converted by qemu-img) and write +it to the block storage disk `/dev/sda`. The raw image is compressed with gzip in this example. -``` +```bash qemu-img convert ubuntu.img ubuntu.raw gzip ubuntu.raw ``` @@ -40,7 +56,7 @@ actions: COMPRESSED: true ``` -## Compression format supported: +## Supported Compression formats - bzip2 (`.bzip2`) - gzip (`.gz`) diff --git a/image2disk/image/image.go b/image2disk/image/image.go index 37dcc81..15a3916 100644 --- a/image2disk/image/image.go +++ b/image2disk/image/image.go @@ -18,7 +18,6 @@ import ( "time" "github.com/klauspost/compress/zstd" - log "github.com/sirupsen/logrus" "github.com/ulikunitz/xz" "golang.org/x/sys/unix" ) @@ -88,8 +87,8 @@ func (wc *WriteCounter) Write(p []byte) (int, error) { // Write will pull an image and write it to local storage device // with compress set to true it will use gzip compression to expand the data before // writing to an underlying device. -func Write(sourceImage, destinationDevice string, compressed bool) error { - req, err := http.NewRequestWithContext(context.TODO(), "GET", sourceImage, nil) +func Write(ctx context.Context, log *slog.Logger, sourceImage, destinationDevice string, compressed bool, progressInterval time.Duration) error { + req, err := http.NewRequestWithContext(ctx, "GET", sourceImage, nil) if err != nil { return err } @@ -101,7 +100,7 @@ func Write(sourceImage, destinationDevice string, compressed bool) error { defer resp.Body.Close() if resp.StatusCode > 300 { - // Customise response for the 404 to make degugging simpler + // Customize response for the 404 to make debugging simpler if resp.StatusCode == 404 { return fmt.Errorf("%s not found", sourceImage) } @@ -131,11 +130,8 @@ func Write(sourceImage, destinationDevice string, compressed bool) error { out = decompressor } - log.Infof("Beginning write of image [%s] to disk [%s]", filepath.Base(sourceImage), destinationDevice) - - log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})) - - ticker := time.NewTicker(time.Second) + log.Info(fmt.Sprintf("Beginning write of image [%s] to disk [%s]", filepath.Base(sourceImage), destinationDevice)) + ticker := time.NewTicker(progressInterval) done := make(chan bool) go func() { totalSize := resp.ContentLength @@ -168,7 +164,7 @@ func Write(sourceImage, destinationDevice string, compressed bool) error { if err := unix.IoctlSetInt(int(fileOut.Fd()), unix.BLKRRPART, 0); err != nil { // Ignore errors since it may be a partition, but log in case it's helpful - log.Error("error re-probing the partitions for the specified device: %v", "err", err) + log.Info("error re-probing the partitions for the specified device", "err", err) } return nil diff --git a/image2disk/main.go b/image2disk/main.go index e3f418e..2a2e623 100644 --- a/image2disk/main.go +++ b/image2disk/main.go @@ -1,44 +1,112 @@ package main import ( + "context" "fmt" "log/slog" + "net/url" "os" + "os/signal" "strconv" + "syscall" "time" "github.com/cenkalti/backoff" + "github.com/lmittmann/tint" + "github.com/mattn/go-isatty" "github.com/tinkerbell/actions/image2disk/image" ) +const ( + defaultRetryDuration = 10 + defaultProgressInterval = 3 +) + func main() { - log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})) - log.Info("IMAGE2DISK - Cloud image streamer") + ctx, done := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGHUP, syscall.SIGTERM) + defer done() + disk := os.Getenv("DEST_DISK") img := os.Getenv("IMG_URL") compressedEnv := os.Getenv("COMPRESSED") + retryEnabled := os.Getenv("RETRY_ENABLED") retryDuration := os.Getenv("RETRY_DURATION_MINUTES") + progressInterval := os.Getenv("PROGRESS_INTERVAL_SECONDS") + textLogging := os.Getenv("TEXT_LOGGING") + + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})) + if tlog, _ := strconv.ParseBool(textLogging); tlog { + w := os.Stderr + log = slog.New(tint.NewHandler(w, &tint.Options{ + NoColor: !isatty.IsTerminal(w.Fd()), + })) + } + + log.Info("IMAGE2DISK - Cloud image streamer") + + if img == "" { + log.Error("IMG_URL is required", "image", img) + os.Exit(1) + } + + if disk == "" { + log.Error("DEST_DISK is required", "disk", disk) + os.Exit(1) + } + u, err := url.Parse(img) + if err != nil { + log.Error("error parsing image URL (IMG_URL)", "err", err, "image", img) + os.Exit(1) + } // We can ignore the error and default compressed to false. cmp, _ := strconv.ParseBool(compressedEnv) + re, _ := strconv.ParseBool(retryEnabled) + pi, err := strconv.Atoi(progressInterval) + if err != nil { + pi = defaultProgressInterval + } + + // convert progress interval to duration in seconds + interval := time.Duration(pi) * time.Second operation := func() error { - if err := image.Write(img, disk, cmp); err != nil { + if err := image.Write(ctx, log, u.String(), disk, cmp, interval); err != nil { return fmt.Errorf("error writing image to disk: %w", err) } return nil } - boff := backoff.NewExponentialBackOff() - rd, err := strconv.Atoi(retryDuration) - if err != nil { - log.Error("error converting retry duration to integer, using 10 minutes for retry duration", "err", err) - rd = 10 - } - boff.MaxElapsedTime = time.Duration(rd) * time.Minute - // try to write the image to disk with exponential backoff for 10 minutes - if err := backoff.Retry(operation, boff); err != nil { - log.Error("error writing image to disk", "err", err) - os.Exit(1) + + if re { + log.Info("retrying of image2disk is enabled") + boff := backoff.NewExponentialBackOff() + rd, err := strconv.Atoi(retryDuration) + if err != nil { + rd = defaultRetryDuration + if retryDuration == "" { + log.Info(fmt.Sprintf("no retry duration specified, using %v minutes for retry duration", rd)) + } else { + log.Info(fmt.Sprintf("error converting retry duration to integer, using %v minutes for retry duration", rd), "err", err) + } + } + boff.MaxElapsedTime = time.Duration(rd) * time.Minute + bctx := backoff.WithContext(boff, ctx) + retryNotifier := func(err error, duration time.Duration) { + log.Error("retrying image2disk", "err", err, "duration", duration) + + } + // try to write the image to disk with exponential backoff for 10 minutes + if err := backoff.RetryNotify(operation, bctx, retryNotifier); err != nil { + log.Error("error writing image to disk", "err", err, "image", img, "disk", disk) + os.Exit(1) + } + } else { + // try to write the image to disk without retry + if err := operation(); err != nil { + log.Error("error writing image to disk", "err", err, "image", img, "disk", disk) + os.Exit(1) + } } + log.Info("Successfully wrote image to disk", "image", img, "disk", disk) } diff --git a/rootio/storage/partition.go b/rootio/storage/partition.go index 2d8300c..f0a0a45 100644 --- a/rootio/storage/partition.go +++ b/rootio/storage/partition.go @@ -95,7 +95,7 @@ func Partition(d Disk) error { case "SWAP": newPartition.Type = gpt.LinuxSwap case "BIOS": - newPartition.Type = gpt.BiosBoot + newPartition.Type = gpt.BIOSBoot case "EFI": newPartition.Type = gpt.EFISystemPartition default: