diff --git a/.github/actions/install-tools/action.yaml b/.github/actions/install-tools/action.yaml index 3e56e2ecef..c4598a19af 100644 --- a/.github/actions/install-tools/action.yaml +++ b/.github/actions/install-tools/action.yaml @@ -4,7 +4,7 @@ description: "Install pipeline tools" runs: using: composite steps: - - uses: sigstore/cosign-installer@c85d0e205a72a294fe064f618a87dbac13084086 # v2.8.1 + - uses: sigstore/cosign-installer@9614fae9e5c5eddabb09f90a270fcb487c9f7149 # v3.3.0 - uses: anchore/sbom-action/download-syft@5ecf649a417b8ae17dc8383dc32d46c03f2312df # v0.15.1 diff --git a/.github/actions/save-logs/action.yaml b/.github/actions/save-logs/action.yaml index 12a1af9e9c..d0d2edf345 100644 --- a/.github/actions/save-logs/action.yaml +++ b/.github/actions/save-logs/action.yaml @@ -1,6 +1,12 @@ name: save-logs description: "Save debug logs" +inputs: + suffix: + description: 'Suffix to append to the debug log' + required: false + default: '' + runs: using: composite steps: @@ -9,7 +15,7 @@ runs: sudo chown $USER /tmp/zarf-*.log || echo "" shell: bash - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + - uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: - name: debug-log + name: debug-log${{ inputs.suffix }} path: /tmp/zarf-*.log diff --git a/.github/workflows/publish-application-packages.yml b/.github/workflows/publish-application-packages.yml index e7979efc9b..6721bd8c46 100644 --- a/.github/workflows/publish-application-packages.yml +++ b/.github/workflows/publish-application-packages.yml @@ -44,9 +44,6 @@ jobs: # Publish a skeleton of the dos-games package zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages - - zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 - zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 env: AWS_REGION: ${{ secrets.COSIGN_AWS_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.COSIGN_AWS_KEY_ID }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3917fe06d6..b9a456b7a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -45,9 +45,10 @@ jobs: docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME . rm build/zarf-linux-amd64 rm build/zarf-linux-arm64 + echo ZARF_AGENT_IMAGE_DIGEST=$(docker buildx imagetools inspect ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME --format '{{ json . }}' | jq -r .manifest.digest) >> $GITHUB_ENV - name: "Zarf Agent: Sign the Image" - run: cosign sign --key awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} -a release-engineer=https://github.com/${{ github.actor }} -a version=$GITHUB_REF_NAME ghcr.io/defenseunicorns/zarf/agent:$GITHUB_REF_NAME + run: cosign sign --key awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} -a release-engineer=https://github.com/${{ github.actor }} -a version=$GITHUB_REF_NAME ghcr.io/defenseunicorns/zarf/agent@$ZARF_AGENT_IMAGE_DIGEST -y env: COSIGN_EXPERIMENTAL: 1 AWS_REGION: ${{ secrets.COSIGN_AWS_REGION }} @@ -71,7 +72,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: build-artifacts path: build/ @@ -88,7 +89,7 @@ jobs: fetch-depth: 0 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -99,7 +100,7 @@ jobs: - name: Make Zarf executable run: | chmod +x build/zarf - + # Before we run the regular tests we need to aggressively cleanup files to reduce disk pressure - name: Cleanup files uses: ./.github/actions/cleanup-files @@ -132,7 +133,7 @@ jobs: uses: ./.github/actions/install-tools - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -158,6 +159,13 @@ jobs: - name: Cleanup files uses: ./.github/actions/cleanup-files + - name: Setup release ENV vars + run: | + K8S_MODULES_VER=$(go list -f '{{.Version}}' -m k8s.io/client-go | sed 's/v//; s/\./ /g') + echo K8S_MODULES_MAJOR_VER=$(expr $(echo "$K8S_MODULES_VER" | cut -d " " -f 1) + 1) >> $GITHUB_ENV + echo K8S_MODULES_MINOR_VER=$(echo "$K8S_MODULES_VER" | cut -d " " -f 2) >> $GITHUB_ENV + echo K8S_MODULES_PATCH_VER=$(echo "$K8S_MODULES_VER" | cut -d " " -f 3) >> $GITHUB_ENV + # Create the GitHub release notes, upload artifact backups to S3, publish homebrew recipe - name: Run GoReleaser uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0 @@ -169,8 +177,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN}} HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.ZARF_ORG_PROJECT_TOKEN }} + - name: Save CVE report - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: cve-report path: build/zarf-known-cves.csv diff --git a/.github/workflows/scan-codeql.yml b/.github/workflows/scan-codeql.yml index 4c990eea61..4ed4534c0d 100644 --- a/.github/workflows/scan-codeql.yml +++ b/.github/workflows/scan-codeql.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@c0d1daa7f7e14667747d73a7dbbe8c074bc8bfe2 # v2.22.9 + uses: github/codeql-action/init@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 env: CODEQL_EXTRACTOR_GO_BUILD_TRACING: on with: @@ -54,6 +54,6 @@ jobs: run: make build-cli-linux-amd - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@c0d1daa7f7e14667747d73a7dbbe8c074bc8bfe2 # v2.22.9 + uses: github/codeql-action/analyze@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index b85c0abc23..f9307bf7e2 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -37,7 +37,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: SARIF file path: results.sarif @@ -45,6 +45,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@c0d1daa7f7e14667747d73a7dbbe8c074bc8bfe2 # v2.22.9 + uses: github/codeql-action/upload-sarif@012739e5082ff0c22ca6d6ab32e07c36df03c4a4 # v3.22.12 with: sarif_file: results.sarif diff --git a/.github/workflows/test-bigbang.yml b/.github/workflows/test-bigbang.yml index bd40729291..4ea00c4d30 100644 --- a/.github/workflows/test-bigbang.yml +++ b/.github/workflows/test-bigbang.yml @@ -53,7 +53,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: build-artifacts path: build/ @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 8a8c5d7f90..9aeb51fe89 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -37,7 +37,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: build-artifacts path: build/ @@ -51,7 +51,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -74,6 +74,8 @@ jobs: - name: Save logs if: always() uses: ./.github/actions/save-logs + with: + suffix: -validate-without-cluster # Run the tests on k3d validate-k3d: @@ -84,7 +86,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -110,6 +112,8 @@ jobs: - name: Save logs if: always() uses: ./.github/actions/save-logs + with: + suffix: -validate-k3d # Run the tests on k3s validate-k3s: @@ -120,7 +124,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -146,6 +150,8 @@ jobs: - name: Save logs if: always() uses: ./.github/actions/save-logs + with: + suffix: -validate-k3s # Run the tests on kind validate-kind: @@ -156,7 +162,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -184,6 +190,8 @@ jobs: - name: Save logs if: always() uses: ./.github/actions/save-logs + with: + suffix: -validate-kind # Run the tests on minikube validate-minikube: @@ -194,7 +202,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ @@ -220,3 +228,5 @@ jobs: - name: Save logs if: always() uses: ./.github/actions/save-logs + with: + suffix: -validate-minikube diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index 1722d3ea36..b5965f9bfc 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -36,7 +36,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 with: name: build-artifacts path: build/ @@ -50,7 +50,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: Download build artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@7a1cd3216ca9260cd8022db641d960b1db4d1be4 # v4.0.0 with: name: build-artifacts path: build/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index ada0dd1ee2..3010314a4a 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -15,7 +15,14 @@ builds: - darwin - windows ldflags: - - -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion={{.Tag}} -X k8s.io/component-base/version.gitVersion=v0.0.0+zarf{{.Tag}} -X k8s.io/component-base/version.gitCommit={{.FullCommit}} -X k8s.io/component-base/version.buildDate={{.Date}} + - -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion={{.Tag}} + - -X k8s.io/component-base/version.gitVersion=v{{.Env.K8S_MODULES_MAJOR_VER}}.{{.Env.K8S_MODULES_MINOR_VER}}.{{.Env.K8S_MODULES_PATCH_VER}} + - -X k8s.io/component-base/version.gitCommit={{.FullCommit}} + - -X k8s.io/component-base/version.buildDate={{.Date}} + - -X helm.sh/helm/v3/pkg/lint/rules.k8sVersionMajor={{.Env.K8S_MODULES_MAJOR_VER}} + - -X helm.sh/helm/v3/pkg/lint/rules.k8sVersionMinor={{.Env.K8S_MODULES_MINOR_VER}} + - -X helm.sh/helm/v3/pkg/chartutil.k8sVersionMajor={{.Env.K8S_MODULES_MAJOR_VER}} + - -X helm.sh/helm/v3/pkg/chartutil.k8sVersionMinor={{.Env.K8S_MODULES_MINOR_VER}} goarch: - amd64 - arm64 diff --git a/Makefile b/Makefile index 3c4d044cf6..6a6b216a01 100644 --- a/Makefile +++ b/Makefile @@ -8,8 +8,10 @@ KEY ?= "" # Figure out which Zarf binary we should use based on the operating system we are on ZARF_BIN := ./build/zarf +BUILD_CLI_FOR_SYSTEM := build-cli-linux-amd ifeq ($(OS),Windows_NT) ZARF_BIN := $(addsuffix .exe,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM := build-cli-windows-amd else UNAME_S := $(shell uname -s) UNAME_P := $(shell uname -p) @@ -19,24 +21,40 @@ else endif ifeq ($(UNAME_P),i386) ZARF_BIN := $(addsuffix -intel,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM = build-cli-mac-intel endif ifeq ($(UNAME_P),arm) ZARF_BIN := $(addsuffix -apple,$(ZARF_BIN)) + BUILD_CLI_FOR_SYSTEM = build-cli-mac-apple endif endif endif CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion") +BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION) +K8S_MODULES_VER=$(subst ., ,$(subst v,,$(shell go list -f '{{.Version}}' -m k8s.io/client-go))) +K8S_MODULES_MAJOR_VER=$(shell echo $$(($(firstword $(K8S_MODULES_VER)) + 1))) +K8S_MODULES_MINOR_VER=$(word 2,$(K8S_MODULES_VER)) +K8S_MODULES_PATCH_VER=$(word 3,$(K8S_MODULES_VER)) + +BUILD_ARGS += -X helm.sh/helm/v3/pkg/lint/rules.k8sVersionMajor=$(K8S_MODULES_MAJOR_VER) +BUILD_ARGS += -X helm.sh/helm/v3/pkg/lint/rules.k8sVersionMinor=$(K8S_MODULES_MINOR_VER) +BUILD_ARGS += -X helm.sh/helm/v3/pkg/chartutil.k8sVersionMajor=$(K8S_MODULES_MAJOR_VER) +BUILD_ARGS += -X helm.sh/helm/v3/pkg/chartutil.k8sVersionMinor=$(K8S_MODULES_MINOR_VER) +BUILD_ARGS += -X k8s.io/component-base/version.gitVersion=v$(K8S_MODULES_MAJOR_VER).$(K8S_MODULES_MINOR_VER).$(K8S_MODULES_PATCH_VER) + GIT_SHA := $(if $(shell git rev-parse HEAD),$(shell git rev-parse HEAD),"") BUILD_DATE := $(shell date -u +'%Y-%m-%dT%H:%M:%SZ') -BUILD_ARGS := -s -w -X 'github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)' -X 'k8s.io/component-base/version.gitVersion=v0.0.0+zarf$(CLI_VERSION)' -X 'k8s.io/component-base/version.gitCommit=$(GIT_SHA)' -X 'k8s.io/component-base/version.buildDate=$(BUILD_DATE)' -.DEFAULT_GOAL := help +BUILD_ARGS += -X k8s.io/component-base/version.gitCommit=$(GIT_SHA) +BUILD_ARGS += -X k8s.io/component-base/version.buildDate=$(BUILD_DATE) + +.DEFAULT_GOAL := build .PHONY: help help: ## Display this help information @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ - | sort | awk 'BEGIN {FS = ":.*?## "}; \ - {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + | sort | awk 'BEGIN {FS = ":.*?## "}; \ + {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' clean: ## Clean the build directory rm -rf build @@ -49,6 +67,9 @@ delete-packages: ## Delete all Zarf package tarballs in the project recursively find . -type f -name 'zarf-package-*' -delete # Note: the path to the main.go file is not used due to https://github.com/golang/go/issues/51831#issuecomment-1074188363 +.PHONY: build +build: ## Build the Zarf CLI for the machines OS and architecture + $(MAKE) $(BUILD_CLI_FOR_SYSTEM) build-cli-linux-amd: ## Build the Zarf CLI for Linux on AMD64 CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="$(BUILD_ARGS)" -o build/zarf . @@ -76,6 +97,9 @@ docs-and-schema: ## Generate the Zarf Documentation and Schema hack/gen-cli-docs.sh ZARF_CONFIG=hack/empty-config.toml hack/create-zarf-schema.sh +lint-packages-and-examples: build ## Recursively lint all zarf.yaml files in the repo except for those dedicated to tests + hack/lint_all_zarf_packages.sh $(ZARF_BIN) + # INTERNAL: a shim used to build the agent image only if needed on Windows using the `test` command init-package-local-agent: @test "$(AGENT_IMAGE_TAG)" != "local" || $(MAKE) build-local-agent-image diff --git a/README.md b/README.md index a5b3518f51..54f61185e1 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Zarf eliminates the [complexity of air gap software delivery](https://www.itopst - Built-in Docker registry - Builtin [K9s Dashboard](https://k9scli.io/) for managing a cluster from the terminal - [Mutating Webhook](adr/0005-mutating-webhook.md) to automatically update Kubernetes pod's image path and pull secrets as well as [Flux Git Repository](https://fluxcd.io/docs/components/source/gitrepositories/) URLs and secret references -- Builtin [command to find images](https://docs.zarf.dev/docs/the-zarf-cli/cli-commands/zarf_prepare_find-images) and resources from a Helm chart +- Builtin [command to find images](https://docs.zarf.dev/docs/the-zarf-cli/cli-commands/zarf_dev_find-images) and resources from a Helm chart - Tunneling capability to [connect to Kubernetes resources](https://docs.zarf.dev/docs/the-zarf-cli/cli-commands/zarf_connect) without network routing, DNS, TLS or Ingress configuration required ## 🛠️ Configurable Features diff --git a/adr/0022-dev-cmd.md b/adr/0022-dev-cmd.md new file mode 100644 index 0000000000..5939aea526 --- /dev/null +++ b/adr/0022-dev-cmd.md @@ -0,0 +1,42 @@ +# 22. Introduce `dev` command + +Date: 2023-12-03 + +## Status + +Accepted + +## Context + +> Feature request: + +The current package development lifecycle is: + +1. Create a `zarf.yaml` file and add components +2. Create a package with `zarf package create ` +3. Debug any create errors and resolve by editing `zarf.yaml` and repeating step 2 +4. Run `zarf init` to initialize the cluster +5. Deploy the package with `zarf package deploy ` +6. Debug any deploy errors and resolve by editing `zarf.yaml` and repeating step 2 or 5 + +If there are deployment errors, the common pattern is to reset the cluster (ex: `k3d cluster delete && k3d cluster create`) and repeat steps 4-6. Re-initializing the cluster, recreating the package, and redeploying the package is tedious and time consuming; especially when the package is large or the change was small. + +`zarf package create` is designed around air-gapped environments where the package is created in one environment and deployed in another. Due to this architecture, a package's dependencies _must_ be retrieved and assembled _each_ and _every_ time. + +There already exists the concept of [`YOLO` mode](0010-yolo-mode.md), which can build + deploy a package without the need for `zarf init`, and builds without fetching certain heavy dependencies (like Docker images). However, `YOLO` mode is not exposed via CLI flags, and is meant to develop and deploy packages in fully connected environments. + +## Decision + +Introduce a `dev deploy` command that will combine the lifecycle of `package create` and `package deploy` into a single command. This command will: + +- Not result in a re-usable tarball / OCI artifact +- Not have any interactive prompts +- Not require `zarf init` to be run by default (override-able with a CLI flag) +- Be able to create+deploy a package in either YOLO mode (default) or prod mode (exposed via a CLI flag) +- Only build + deploy components that _will_ be deployed (contrasting with `package create` which builds _all_ components regardless of whether they will be deployed) + +## Consequences + +The `dev deploy` command will make it easier to develop and deploy packages in connected **development** environments. It will also make it easier to debug deployment errors by allowing the user to iterate on the package without having to re-initialize the cluster, re-build and re-deploy the package each cycle. + +There is a purpose to placing this functionality behind a new command, rather than adding it to `package create`. Commands under `dev` are meant to be used in **development** environments, and are **not** meant to be used in **production** environments. There is still the possibility that a user will use `dev deploy` in a production environment, but the command name and documentation will make it clear that this is not the intended use case. diff --git a/docs-website/static/docs/tutorials/publish_and_deploy_deploy.html b/docs-website/static/docs/tutorials/publish_and_deploy_deploy.html index 011823eacd..6bec7c1c49 100644 --- a/docs-website/static/docs/tutorials/publish_and_deploy_deploy.html +++ b/docs-website/static/docs/tutorials/publish_and_deploy_deploy.html @@ -47,11 +47,11 @@
-$ zarf package deploy oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+$ zarf package deploy oci://$REPOSITORY_URL/helm-oci-chart:0.0.1
 
 Saving log file to
 /var/folders/bk/rz1xx2sd5zn134c0_j1s2n5r0000gp/T/zarf-2023-03-30-12-09-38-2083571763.log
-Pulling Zarf package from $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+Pulling Zarf package from $REPOSITORY_URL/helm-oci-chart:0.0.1Pulling Zarf package data (0.00 Byte of 26.90 MBs)
  d8399f7b56ca [application/vnd.unknown.config.v1+json]
  fd143c92d486 zarf.yaml
@@ -70,7 +70,7 @@
  8c5b695f4724 images/blobs/sha256/8c5b695...014f94c8d4ea62772c477c1e03
  cf79ae90993d [application/vnd.oci.image.manifest.v1+json]
  Pulling Zarf package data (26.90 MBs)
- Pulled $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+ Pulled $REPOSITORY_URL/helm-oci-chart:0.0.1Loading Zarf Package /var/folders/bk/rz1xx2sd5zn134c0_j1s2n5r0000gp/T/zarf-3635611772Loading Zarf Package /var/folders/bk/rz1xx2sd5zn134c0_j1s2n5r0000gp/T/zarf-3635611772
 
diff --git a/docs-website/static/docs/tutorials/publish_and_deploy_inspect.html b/docs-website/static/docs/tutorials/publish_and_deploy_inspect.html
index 7e80c7a128..e32ecd4d1d 100644
--- a/docs-website/static/docs/tutorials/publish_and_deploy_inspect.html
+++ b/docs-website/static/docs/tutorials/publish_and_deploy_inspect.html
@@ -47,12 +47,12 @@
 
 
 
-$ zarf package inspect oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+$ zarf package inspect oci://$REPOSITORY_URL/helm-oci-chart:0.0.1
 
 Saving log file to
 /var/folders/bk/rz1xx2sd5zn134c0_j1s2n5r0000gp/T/zarf-2023-03-30-13-13-29-2403571657.log
-Loading Zarf Package oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
-  •  Loaded Zarf Package oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+Loading Zarf Package oci://$REPOSITORY_URL/helm-oci-chart:0.0.1
+  •  Loaded Zarf Package oci://$REPOSITORY_URL/helm-oci-chart:0.0.1
 
 
 kind: ZarfPackageConfig
diff --git a/docs-website/static/docs/tutorials/publish_and_deploy_publish.html b/docs-website/static/docs/tutorials/publish_and_deploy_publish.html
index b2dc87096f..ee95783c43 100644
--- a/docs-website/static/docs/tutorials/publish_and_deploy_publish.html
+++ b/docs-website/static/docs/tutorials/publish_and_deploy_publish.html
@@ -56,15 +56,15 @@
 
 
- 📦 PACKAGE PUBLISH helm-oci-chart:0.0.1-arm64 + 📦 PACKAGE PUBLISH helm-oci-chart:0.0.1
- • Publishing package to $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64 + • Publishing package to $REPOSITORY_URL/helm-oci-chart:0.0.1Prepared 14 layers - • Publishing jvb/helm-oci-chart:0.0.1-arm64 + • Publishing jvb/helm-oci-chart:0.0.1 b66dbb27a733 images/oci-layout 515aceaacb8d images/index.json fd143c92d486 zarf.yaml @@ -79,7 +79,7 @@ b95c82728c36 images/blobs/sha256/b95c827...042a9c5d84426c1674044916d4 e2b45cdcd8bf images/blobs/sha256/e2b45cd...000f1bc1695014e38821dc675c 42c097bd02de components/helm-oci-chart.tar - • Publishing jvb/helm-oci-chart:0.0.1-arm64 + • Publishing jvb/helm-oci-chart:0.0.1 d8399f7b56ca [application/vnd.unknown.config.v1+json] 515aceaacb8d images/index.json b66dbb27a733 images/oci-layout @@ -96,12 +96,12 @@ ab67ffd6e92e images/blobs/sha256/ab67ffd...f8c9d93c0e719f6350e99d3aea e2b45cdcd8bf images/blobs/sha256/e2b45cd...000f1bc1695014e38821dc675c cf79ae90993d [application/vnd.oci.image.manifest.v1+json] - Published $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64 [application/vnd.oci.image.manifest.v1+json] + Published $REPOSITORY_URL/helm-oci-chart:0.0.1 [application/vnd.oci.image.manifest.v1+json]To inspect/deploy/pull: - • zarf package inspect oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64 --insecure - • zarf package deploy oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64 --insecure - • zarf package pull oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64 --insecure + • zarf package inspect oci://$REPOSITORY_URL/helm-oci-chart:0.0.1 --insecure + • zarf package deploy oci://$REPOSITORY_URL/helm-oci-chart:0.0.1 --insecure + • zarf package pull oci://$REPOSITORY_URL/helm-oci-chart:0.0.1 --insecure
diff --git a/docs-website/static/docs/tutorials/publish_and_deploy_pull.html b/docs-website/static/docs/tutorials/publish_and_deploy_pull.html index 561176211a..594d56f42b 100644 --- a/docs-website/static/docs/tutorials/publish_and_deploy_pull.html +++ b/docs-website/static/docs/tutorials/publish_and_deploy_pull.html @@ -47,11 +47,11 @@
-$ zarf package pull oci://$REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+$ zarf package pull oci://$REPOSITORY_URL/helm-oci-chart:0.0.1
 
 Saving log file to
 /var/folders/bk/rz1xx2sd5zn134c0_j1s2n5r0000gp/T/zarf-2023-03-30-11-56-13-4058959193.log
-Pulling Zarf package from $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+Pulling Zarf package from $REPOSITORY_URL/helm-oci-chart:0.0.1Pulling Zarf package data (0.00 Byte of 26.90 MBs)
  515aceaacb8d images/index.json
  fd143c92d486 zarf.yaml
@@ -70,7 +70,7 @@
  8c5b695f4724 images/blobs/sha256/8c5b695...014f94c8d4ea62772c477c1e03
  cf79ae90993d [application/vnd.oci.image.manifest.v1+json]
  Pulling Zarf package data (26.90 MBs)
- Pulled $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64
+ Pulled $REPOSITORY_URL/helm-oci-chart:0.0.1
 
 # Use vim if you want to inspect the tarball's contents without decompressing it
 $ vim zarf-package-helm-oci-chart-arm64-0.0.1.tar.zst
diff --git a/docs-website/static/docs/tutorials/troubleshoot_insecure_registry.html b/docs-website/static/docs/tutorials/troubleshoot_insecure_registry.html
index 5a87cd8d59..6f8ffcd330 100644
--- a/docs-website/static/docs/tutorials/troubleshoot_insecure_registry.html
+++ b/docs-website/static/docs/tutorials/troubleshoot_insecure_registry.html
@@ -56,16 +56,16 @@
 
 
- 📦 PACKAGE PUBLISH helm-oci-chart:0.0.1-arm64 + 📦 PACKAGE PUBLISH helm-oci-chart:0.0.1
- • Publishing package to $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64 + • Publishing package to $REPOSITORY_URL/helm-oci-chart:0.0.1Prepared 14 layers - • Publishing jvb/helm-oci-chart:0.0.1-arm64 - ERROR: Failed to publish package: unable to publish package $REPOSITORY_URL/helm-oci-chart:0.0.1-arm64: + • Publishing jvb/helm-oci-chart:0.0.1 + ERROR: Failed to publish package: unable to publish package $REPOSITORY_URL/helm-oci-chart:0.0.1: Head "https://localhost:5000/v2/jvb/helm-oci-chart/manifests/sha256:239ff96a91a4e356ed6e8eadb4dad633cbffea69fc35d436e08ebdc7f2d9a2fd": http: server gave HTTP response to HTTPS client diff --git a/docs/0-zarf-overview.md b/docs/0-zarf-overview.md index d9234c1d50..98d2b5f97a 100644 --- a/docs/0-zarf-overview.md +++ b/docs/0-zarf-overview.md @@ -148,7 +148,7 @@ In this use case, you configure Zarf to initialize a cluster that already exists - Builtin Docker registry - Builtin [K9s Dashboard](https://k9scli.io/) for managing a cluster from the terminal - [Mutating Webhook](adr/0005-mutating-webhook.md) to automatically update Kubernetes pod's image path and pull secrets as well as [Flux Git Repository](https://fluxcd.io/docs/components/source/gitrepositories/) URLs and secret references -- Builtin [command to find images](./2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md) and resources from a Helm chart +- Builtin [command to find images](./2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md) and resources from a Helm chart - Tunneling capability to [connect to Kuberenetes resources](./2-the-zarf-cli/100-cli-commands/zarf_connect.md) without network routing, DNS, TLS or Ingress configuration required ### 🛠️ Configurable Features diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf.md b/docs/2-the-zarf-cli/100-cli-commands/zarf.md index 5fa846e461..f2794aa98e 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf.md @@ -31,8 +31,8 @@ zarf COMMAND [flags] * [zarf completion](zarf_completion.md) - Generate the autocompletion script for the specified shell * [zarf connect](zarf_connect.md) - Accesses services or pods deployed in the cluster * [zarf destroy](zarf_destroy.md) - Tears down Zarf and removes its components from the environment +* [zarf dev](zarf_dev.md) - Commands useful for developing packages * [zarf init](zarf_init.md) - Prepares a k8s cluster for the deployment of Zarf packages * [zarf package](zarf_package.md) - Zarf package commands for creating, deploying, and inspecting packages -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging * [zarf tools](zarf_tools.md) - Collection of additional tools to make airgap easier * [zarf version](zarf_version.md) - Shows the version of the running Zarf binary diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md similarity index 59% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md index 837e4c03ba..0e44236b00 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md @@ -1,12 +1,12 @@ -# zarf prepare +# zarf dev -Tools to help prepare assets for packaging +Commands useful for developing packages ## Options ``` - -h, --help help for prepare + -h, --help help for dev ``` ## Options inherited from parent commands @@ -25,9 +25,10 @@ Tools to help prepare assets for packaging ## SEE ALSO * [zarf](zarf.md) - DevSecOps for Airgap -* [zarf prepare find-images](zarf_prepare_find-images.md) - Evaluates components in a zarf file to identify images specified in their helm charts and manifests -* [zarf prepare generate-config](zarf_prepare_generate-config.md) - Generates a config file for Zarf -* [zarf prepare lint](zarf_prepare_lint.md) - Verifies the package schema -* [zarf prepare patch-git](zarf_prepare_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: +* [zarf dev deploy](zarf_dev_deploy.md) - [beta] Creates and deploys a Zarf package from a given directory +* [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests +* [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf +* [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices +* [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. -* [zarf prepare sha256sum](zarf_prepare_sha256sum.md) - Generates a SHA256SUM for the given file +* [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md new file mode 100644 index 0000000000..36961236b3 --- /dev/null +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_deploy.md @@ -0,0 +1,41 @@ +# zarf dev deploy + + +[beta] Creates and deploys a Zarf package from a given directory + +## Synopsis + +[beta] Creates and deploys a Zarf package from a given directory, setting options like YOLO mode for faster iteration. + +``` +zarf dev deploy [flags] +``` + +## Options + +``` + --components string Comma-separated list of components to deploy. Adding this flag will skip the prompts for selected components. Globbing component names with '*' and deselecting 'default' components with a leading '-' are also supported. + --create-set stringToString Specify package variables to set on the command line (KEY=value) (default []) + --deploy-set stringToString Specify deployment variables to set on the command line (KEY=value) (default []) + -f, --flavor string The flavor of components to include in the resulting package (i.e. have a matching or empty "only.flavor" key) + -h, --help help for deploy + --no-yolo Disable the YOLO mode default override and create / deploy the package as-defined + --registry-override stringToString Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default []) +``` + +## Options inherited from parent commands + +``` + -a, --architecture string Architecture for OCI images and Zarf packages + --insecure Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture. + -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info") + --no-color Disable colors in output + --no-log-file Disable log file creation + --no-progress Disable fancy UI progress bars, spinners, logos, etc + --tmpdir string Specify the temporary directory to use for intermediate files + --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache") +``` + +## SEE ALSO + +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md similarity index 85% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md index 6e2a1d84e4..01e43dfbcb 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md @@ -1,16 +1,16 @@ -# zarf prepare find-images +# zarf dev find-images -Evaluates components in a zarf file to identify images specified in their helm charts and manifests +Evaluates components in a Zarf file to identify images specified in their helm charts and manifests ## Synopsis -Evaluates components in a zarf file to identify images specified in their helm charts and manifests. +Evaluates components in a Zarf file to identify images specified in their helm charts and manifests. Components that have repos that host helm charts can be processed by providing the --repo-chart-path. ``` -zarf prepare find-images [ PACKAGE ] [flags] +zarf dev find-images [ PACKAGE ] [flags] ``` ## Options @@ -37,4 +37,4 @@ zarf prepare find-images [ PACKAGE ] [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_generate-config.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_generate-config.md similarity index 90% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_generate-config.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_generate-config.md index 6a12b68380..4005e20995 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_generate-config.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_generate-config.md @@ -1,4 +1,4 @@ -# zarf prepare generate-config +# zarf dev generate-config Generates a config file for Zarf @@ -13,7 +13,7 @@ Accepted extensions are json, toml, yaml. NOTE: This file must not already exist. If no filename is provided, the config will be written to the current working directory as zarf-config.toml. ``` -zarf prepare generate-config [ FILENAME ] [flags] +zarf dev generate-config [ FILENAME ] [flags] ``` ## Options @@ -37,4 +37,4 @@ zarf prepare generate-config [ FILENAME ] [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md similarity index 63% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md index bb36206a1d..7ec1068b0b 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_lint.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_lint.md @@ -1,20 +1,22 @@ -# zarf prepare lint +# zarf dev lint -Verifies the package schema +Lints the given package for valid schema and recommended practices ## Synopsis -Verifies the package schema and warns the user if they have variables that won't be evaluated +Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files ``` -zarf prepare lint [ DIRECTORY ] [flags] +zarf dev lint [ DIRECTORY ] [flags] ``` ## Options ``` - -h, --help help for lint + -f, --flavor string The flavor of components to include in the resulting package (i.e. have a matching or empty "only.flavor" key) + -h, --help help for lint + --set stringToString Specify package variables to set on the command line (KEY=value) (default []) ``` ## Options inherited from parent commands @@ -32,4 +34,4 @@ zarf prepare lint [ DIRECTORY ] [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_patch-git.md similarity index 90% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_patch-git.md index 022e93eb72..808df2ba07 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_patch-git.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_patch-git.md @@ -1,11 +1,11 @@ -# zarf prepare patch-git +# zarf dev patch-git Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE: This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook. ``` -zarf prepare patch-git HOST FILE [flags] +zarf dev patch-git HOST FILE [flags] ``` ## Options @@ -30,4 +30,4 @@ zarf prepare patch-git HOST FILE [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_sha256sum.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_sha256sum.md similarity index 89% rename from docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_sha256sum.md rename to docs/2-the-zarf-cli/100-cli-commands/zarf_dev_sha256sum.md index 4b64ade810..4be9cbcd30 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_prepare_sha256sum.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_dev_sha256sum.md @@ -1,10 +1,10 @@ -# zarf prepare sha256sum +# zarf dev sha256sum Generates a SHA256SUM for the given file ``` -zarf prepare sha256sum { FILE | URL } [flags] +zarf dev sha256sum { FILE | URL } [flags] ``` ## Options @@ -29,4 +29,4 @@ zarf prepare sha256sum { FILE | URL } [flags] ## SEE ALSO -* [zarf prepare](zarf_prepare.md) - Tools to help prepare assets for packaging +* [zarf dev](zarf_dev.md) - Commands useful for developing packages diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md index 5dc213e8e1..960898294b 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_deploy.md @@ -16,7 +16,7 @@ zarf package deploy [ PACKAGE_SOURCE ] [flags] ``` --adopt-existing-resources Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover. - --components string Comma-separated list of components to install. Adding this flag will skip the init prompts for which components to install + --components string Comma-separated list of components to deploy. Adding this flag will skip the prompts for selected components. Globbing component names with '*' and deselecting 'default' components with a leading '-' are also supported. --confirm Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes. -h, --help help for deploy --set stringToString Specify deployment variables to set on the command line (KEY=value) (default []) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md index 32b7ffc4ae..2493836807 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_mirror-resources.md @@ -5,7 +5,7 @@ Mirrors a Zarf package's internal resources to specified image registries and gi ## Synopsis -Unpacks resources and dependencies from a Zarf package archive and mirrors them into the specified +Unpacks resources and dependencies from a Zarf package archive and mirrors them into the specified image registries and git repositories within the target environment ``` @@ -39,7 +39,7 @@ $ zarf package mirror-resources \ ## Options ``` - --components string Comma-separated list of components to mirror. This list will be respected regardless of a component's 'required' status. + --components string Comma-separated list of components to mirror. This list will be respected regardless of a component's 'required' or 'default' status. Globbing component names with '*' and deselecting components with a leading '-' are also supported. --confirm Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes. --git-push-password string Password for the push-user to access the git server --git-push-username string Username to access to the git server Zarf is configured to use. User must be able to create repositories via 'git push' (default "zarf-git-user") diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md index 7693c9d7b3..59d876923c 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_pull.md @@ -10,7 +10,15 @@ zarf package pull PACKAGE_SOURCE [flags] ## Examples ``` -$ zarf package pull oci://my-registry.com/my-namespace/my-package:0.0.1-arm64 + +# Pull a package matching the current architecture +$ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 + +# Pull a package matching a specific architecture +$ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a arm64 + +# Pull a skeleton package +$ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a skeleton ``` ## Options diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md index 022294b49a..3e4a20ae08 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_remove.md @@ -10,7 +10,7 @@ zarf package remove { PACKAGE_SOURCE | PACKAGE_NAME } --confirm [flags] ## Options ``` - --components string Comma-separated list of components to uninstall + --components string Comma-separated list of components to remove. This list will be respected regardless of a component's 'required' or 'default' status. Globbing component names with '*' and deselecting components with a leading '-' are also supported. --confirm REQUIRED. Confirm the removal action to prevent accidental deletions -h, --help help for remove ``` diff --git a/docs/2-the-zarf-cli/2-zarf-config-files.md b/docs/2-the-zarf-cli/2-zarf-config-files.md index 9aa9852fd7..650abaed3a 100644 --- a/docs/2-the-zarf-cli/2-zarf-config-files.md +++ b/docs/2-the-zarf-cli/2-zarf-config-files.md @@ -8,7 +8,7 @@ import FetchFileCodeBlock from '@site/src/components/FetchFileCodeBlock'; Users can use a config file to easily control flags for `zarf init`, `zarf package create`, and `zarf package deploy` commands, as well as global flags (excluding `--confirm`), enabling a more straightforward and declarative workflow. -Zarf supports config files written in common configuration file formats including `toml`, `json`, `yaml`, `ini` and `props`, and by default Zarf will look for a file called `zarf-config` with one of these filenames in the current working directory. To generate a blank config file you can run `zarf prepare generate-config` with an optional output filename/format. For example, to create an empty config file with the `my-cool-env` in the yaml format, you can use `zarf prepare generate-config my-cool-env.yaml`. +Zarf supports config files written in common configuration file formats including `toml`, `json`, `yaml`, `ini` and `props`, and by default Zarf will look for a file called `zarf-config` with one of these filenames in the current working directory. To generate a blank config file you can run `zarf dev generate-config` with an optional output filename/format. For example, to create an empty config file with the `my-cool-env` in the yaml format, you can use `zarf dev generate-config my-cool-env.yaml`. To use a custom config filename, set the `ZARF_CONFIG` environment variable to the config file's path. For example, to use the `my-cool-env.yaml` config file in the current working directory, you can set the `ZARF_CONFIG` environment variable to `my-cool-env.yaml`. The `ZARF_CONFIG` environment variable can be set either in the shell or in a `.env` file in the current working directory. Note that the `ZARF_CONFIG` environment variable takes precedence over the default config file path. diff --git a/docs/2-the-zarf-cli/index.md b/docs/2-the-zarf-cli/index.md index 3992d368e8..0e774c88fb 100644 --- a/docs/2-the-zarf-cli/index.md +++ b/docs/2-the-zarf-cli/index.md @@ -34,6 +34,6 @@ The `zarf package create` command is used to create a Zarf package from a `zarf. :::tip -When deploying and managing packages you may find the sub-commands under `zarf prepare` useful to find resources and manipulate package definitions as needed. +When developing packages you may find the sub-commands under `zarf dev` useful to find resources and manipulate package definitions. ::: diff --git a/docs/3-create-a-zarf-package/1-zarf-packages.md b/docs/3-create-a-zarf-package/1-zarf-packages.md index 6595ac1bef..610b3c14d7 100644 --- a/docs/3-create-a-zarf-package/1-zarf-packages.md +++ b/docs/3-create-a-zarf-package/1-zarf-packages.md @@ -68,6 +68,7 @@ When executed, the `zarf package create` command locates the `zarf.yaml` file in The process of defining and creating a package is also elaborated on in detail in the [Creating a Zarf Package Tutorial](../5-zarf-tutorials/0-creating-a-zarf-package.md). ### Creating Differential Packages + If you already have a Zarf package and you want to create an updated package you would normally have to re-create the entire package from scratch, including things that might not have changed. Depending on your workflow, you may want to create a package that only contains the artifacts that have changed since the last time you built your package. This can be achieved by using the `--differential` flag while running the `zarf package create` command. You can use this flag to point to an already built package you have locally or to a package that has been previously [published](../5-zarf-tutorials/7-publish-and-deploy.md#publish-package) to a registry. ## Inspecting a Created Package diff --git a/docs/3-create-a-zarf-package/10-dev.md b/docs/3-create-a-zarf-package/10-dev.md new file mode 100644 index 0000000000..9f7b132887 --- /dev/null +++ b/docs/3-create-a-zarf-package/10-dev.md @@ -0,0 +1,57 @@ +# Developing Zarf Packages + +## `dev` Commands + +Zarf contains many commands that are useful while developing a Zarf package to iterate on configuration, discover resources and more! Below are explanations of some of these commands with the full list discoverable with `zarf dev --help`. + +:::caution + +The `dev` commands are meant to be used in **development** environments / workflows. They are **not** meant to be used in **production** environments / workflows. + +::: + +### `dev deploy` + +The `dev deploy` command will combine the lifecycle of `package create` and `package deploy` into a single command. This command will: + +- Not result in a re-usable tarball / OCI artifact +- Not have any interactive prompts +- Not require `zarf init` to be run (by default, but _is required_ if `--no-yolo` is not set) +- Be able to create+deploy a package in either YOLO mode (default) or prod mode (exposed via `--no-yolo` flag) +- Only build + deploy components that _will_ be deployed (contrasting with `package create` which builds _all_ components regardless of whether they will be deployed) + +```bash +# Create and deploy dos-games in yolo mode +$ zarf dev deploy examples/dos-games +``` + +```bash +# If deploying a package in prod mode, `zarf init` must be run first +$ zarf init --confirm +# Create and deploy dos-games in prod mode +$ zarf dev deploy examples/dos-games --no-yolo +``` + +### `dev find-images` + +Evaluates components in a `zarf.yaml` to identify images specified in their helm charts and manifests. + +Components that have `git` repositories that host helm charts can be processed by providing the `--repo-chart-path`. + +```bash +$ zarf dev find-images examples/wordpress + +components: + + - name: wordpress + images: + - docker.io/bitnami/apache-exporter:0.13.3-debian-11-r2 + - docker.io/bitnami/mariadb:10.11.2-debian-11-r21 + - docker.io/bitnami/wordpress:6.2.0-debian-11-r18 +``` + +### Misc `dev` Commands + +Not all `dev` commands have been mentioned here. + +Further `dev` commands can be discovered by running `zarf dev --help`. diff --git a/docs/3-create-a-zarf-package/2-zarf-components.md b/docs/3-create-a-zarf-package/2-zarf-components.md index df26966412..1ccd87dcc1 100644 --- a/docs/3-create-a-zarf-package/2-zarf-components.md +++ b/docs/3-create-a-zarf-package/2-zarf-components.md @@ -116,11 +116,11 @@ Kustomizations are handled a bit differently than normal manifests in that Zarf -Images can either be discovered manually, or automatically by using [`zarf prepare find-images`](../2-the-zarf-cli/100-cli-commands/zarf_prepare_find-images.md). +Images can either be discovered manually, or automatically by using [`zarf dev find-images`](../2-the-zarf-cli/100-cli-commands/zarf_dev_find-images.md). :::note -`zarf prepare find-images` will find images for most standard manifests, kustomizations, and helm charts, however some images cannot be discovered this way as some upstream resources (like operators) may bury image definitions inside. For these images, `zarf prepare find-images` also offers support for the draft [Helm Improvement Proposal 15](https://github.com/helm/community/blob/main/hips/hip-0015.md) which allows chart creators to annotate any hidden images in their charts along with the [values conditions](https://github.com/helm/community/issues/277) that will cause those images to be used. +`zarf dev find-images` will find images for most standard manifests, kustomizations, and helm charts, however some images cannot be discovered this way as some upstream resources (like operators) may bury image definitions inside. For these images, `zarf dev find-images` also offers support for the draft [Helm Improvement Proposal 15](https://github.com/helm/community/blob/main/hips/hip-0015.md) which allows chart creators to annotate any hidden images in their charts along with the [values conditions](https://github.com/helm/community/issues/277) that will cause those images to be used. ::: @@ -204,3 +204,21 @@ $ zarf package deploy ./path/to/package.tar.zst --confirm # deploy optional-component-1 and optional-component-2 components whether they are required or not $ zarf package deploy ./path/to/package.tar.zst --components=optional-component-1,optional-component-2 ``` + +:::tip + +You can deploy components in a package using globbing as well. The following would deploy all components regardless of optional status: + +```bash +# deploy optional-component-1 and optional-component-2 components whether they are required or not +$ zarf package deploy ./path/to/package.tar.zst --components=* +``` + +If you have any `default` components in a package definition you can also exclude those from the CLI with a leading dash (`-`) (similar to how you can exclude search terms in a search engine). + +```bash +# deploy optional-component-1 but exclude default-component-1 +$ zarf package deploy ./path/to/package.tar.zst --components=optional-component-1,-default-component-1 +``` + +::: diff --git a/docs/3-create-a-zarf-package/3-zarf-init-package.md b/docs/3-create-a-zarf-package/3-zarf-init-package.md index f402349786..f405193893 100644 --- a/docs/3-create-a-zarf-package/3-zarf-init-package.md +++ b/docs/3-create-a-zarf-package/3-zarf-init-package.md @@ -72,7 +72,7 @@ There are two ways to deploy these optional components. First, you can provide a The `k3s` component included in Zarf differs from the default `k3s` install in that it disables the installation of `traefik` out of the box. This was done so that people could more intentionally choose if they wanted `traefik` or another ingress provider (or no ingress at all) depending on their needs. If you would like to return `k3s` to its defaults, you can set the `K3S_ARGS` zarf variable to an empty string: -``` +```text root@machine ~ # zarf init --components k3s --set K3S_ARGS="" --confirm ``` diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 6f6ae55236..8962ec9fc6 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -63,9 +63,9 @@ Must be one of: | -------- | -------- | | **Type** | `string` | -| Restrictions | | -| --------------------------------- | --------------------------------------------------------------------------------- | -| **Must match regular expression** | ```^[a-z0-9\-]+$``` [Test](https://regex101.com/?regex=%5E%5Ba-z0-9%5C-%5D%2B%24) | +| Restrictions | | +| --------------------------------- | ----------------------------------------------------------------------------------------------------- | +| **Must match regular expression** | ```^[a-z0-9\-]*[a-z0-9]$``` [Test](https://regex101.com/?regex=%5E%5Ba-z0-9%5C-%5D%2A%5Ba-z0-9%5D%24) | @@ -507,6 +507,22 @@ must respect the following conditions +
+ + flavor + +  +
+ +**Description:** The flavor of Zarf used to build this package + +| | | +| -------- | -------- | +| **Type** | `string` | + +
+
+ @@ -554,9 +570,9 @@ must respect the following conditions | -------- | -------- | | **Type** | `string` | -| Restrictions | | -| --------------------------------- | --------------------------------------------------------------------------------- | -| **Must match regular expression** | ```^[a-z0-9\-]+$``` [Test](https://regex101.com/?regex=%5E%5Ba-z0-9%5C-%5D%2B%24) | +| Restrictions | | +| --------------------------------- | ----------------------------------------------------------------------------------------------------- | +| **Must match regular expression** | ```^[a-z0-9\-]*[a-z0-9]$``` [Test](https://regex101.com/?regex=%5E%5Ba-z0-9%5C-%5D%2A%5Ba-z0-9%5D%24) | diff --git a/docs/3-create-a-zarf-package/6-package-sboms.md b/docs/3-create-a-zarf-package/6-package-sboms.md index cc84e4244a..f9826f0374 100644 --- a/docs/3-create-a-zarf-package/6-package-sboms.md +++ b/docs/3-create-a-zarf-package/6-package-sboms.md @@ -29,6 +29,7 @@ Zarf uses the `file:` Syft SBOM scheme even if given a directory as the `files` Given the Syft CLI is vendored into Zarf you can run these commands with the Zarf binary as well: ```bash +# Syft is vendored as `zarf tools sbom` $ zarf tools sbom packages file:path/to/yourproject/file -o json > my-sbom.json ``` diff --git a/docs/3-create-a-zarf-package/8-vscode.md b/docs/3-create-a-zarf-package/8-vscode.md index 79e10ce663..896c898d3c 100644 --- a/docs/3-create-a-zarf-package/8-vscode.md +++ b/docs/3-create-a-zarf-package/8-vscode.md @@ -15,6 +15,7 @@ Zarf uses the [Zarf package schema](https://github.com/defenseunicorns/zarf/blob "https://raw.githubusercontent.com/defenseunicorns/zarf/main/zarf.schema.json": "zarf.yaml" } ``` + :::note When successfully installed, the `yaml.schema` line will match the color of the other lines within the settings. diff --git a/docs/3-create-a-zarf-package/index.md b/docs/3-create-a-zarf-package/index.md index 692f37a9db..ce5f4f6ff8 100644 --- a/docs/3-create-a-zarf-package/index.md +++ b/docs/3-create-a-zarf-package/index.md @@ -16,13 +16,13 @@ To learn more about creating a Zarf package, you can check out the following res - [The Package Create Lifecycle](./5-package-create-lifecycle.md): An overview of the lifecycle of `zarf package create`. - [Creating a Zarf Package Tutorial](../5-zarf-tutorials/0-creating-a-zarf-package.md): A tutorial covering how to take an application and create a package for it. -## Typical Creation Workflow: +## Typical Creation Workflow The general flow of a Zarf package deployment on an existing initialized cluster is as follows: ```shell # Before creating your package you can lint your zarf.yaml -$ zarf prepare lint +$ zarf dev lint # To create a package run the following: $ zarf package create diff --git a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md index f311a4d387..e398df6933 100644 --- a/docs/5-zarf-tutorials/0-creating-a-zarf-package.md +++ b/docs/5-zarf-tutorials/0-creating-a-zarf-package.md @@ -2,7 +2,7 @@ ## Introduction -In this tutorial, we will demonstrate the process to create a Zarf package for an application from defining a `zarf.yaml`, finding resources with `zarf prepare` commands and finally building the package with `zarf package create`. +In this tutorial, we will demonstrate the process to create a Zarf package for an application from defining a `zarf.yaml`, finding resources with `zarf dev` commands and finally building the package with `zarf package create`. When creating a Zarf package, you must have a network connection so that Zarf can fetch all of the dependencies and resources necessary to build the package. If your package is using images from a private registry or is referencing repositories in a private repository, you will need to have your credentials configured on your machine for Zarf to be able to fetch the resources. @@ -37,7 +37,7 @@ metadata: :::tip If you are using an Integrated Development Environment (such as [VS Code](../3-create-a-zarf-package/8-vscode.md)) to create and edit the `zarf.yaml` file, you can install or reference the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) file to get error checking and autocomplete. -Additionally, you can run `zarf prepare lint ` to validate aginst the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) +Additionally, you can run `zarf dev lint ` to validate aginst the [`zarf.schema.json`](https://github.com/defenseunicorns/zarf/blob/main/zarf.schema.json) ::: @@ -83,7 +83,7 @@ service: :::note -We create any `values.yaml` file(s) at this stage because the `zarf prepare find-images` command we will use next will template out this chart to look only for the images we need. +We create any `values.yaml` file(s) at this stage because the `zarf dev find-images` command we will use next will template out this chart to look only for the images we need. ::: @@ -95,7 +95,7 @@ Note that we are explicitly defining the `wordpress` namespace for this deployme ### Finding the Images -Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf prepare find-images`. Running this command in the directory of your zarf.yaml will result in the following output: +Once you have the above defined we can now work on setting the images that we will need to bring with us into the air gap. For this, Zarf has a helper command you can run with `zarf dev find-images`. Running this command in the directory of your zarf.yaml will result in the following output: @@ -109,7 +109,7 @@ Due to the way some applications are deployed, Zarf might not be able to find al :::tip -Zarf has more `prepare` commands you can learn about on the [prepare CLI docs page](../2-the-zarf-cli/100-cli-commands/zarf_prepare.md). +Zarf has more `dev` commands you can learn about on the [dev CLI docs page](../3-create-a-zarf-package/10-dev.md). ::: diff --git a/docs/5-zarf-tutorials/8-custom-init-packages.md b/docs/5-zarf-tutorials/8-custom-init-packages.md index 6f2e455b68..a99ebc1b59 100644 --- a/docs/5-zarf-tutorials/8-custom-init-packages.md +++ b/docs/5-zarf-tutorials/8-custom-init-packages.md @@ -91,11 +91,10 @@ $ zarf package create . \ --set REGISTRY_IMAGE_TAG=2.8.3 \ --set REGISTRY_IMAGE="opensource/registry" \ --set REGISTRY_IMAGE_DOMAIN="custom.enterprise.corp" \ ---set GITEA_IMAGE="custom.enterprise.corp/opensource/gitea" \ ---set GITEA_SERVER_VERSION="v1.19.3" +--set GITEA_IMAGE="custom.enterprise.corp/opensource/gitea:v1.21.0-rootless" ``` -⚠️ - The Gitea image and version are different than the Agent and Registry in that Zarf will always prefer the `rootless` version of a given server image. This means that the above reference would template out to be `custom.enterprise.corp/opensource/gitea:v1.19.3-rootless`. If you need to change this, edit the `packages/gitea` package. +⚠️ - The Gitea image is different from the Agent and Registry in that Zarf will always prefer the `rootless` version of a given server image. The image no longer must be tagged with `-rootless`, but it still needs to implement the [Gitea configuration of a rootless image](https://github.com/go-gitea/gitea/blob/main/Dockerfile.rootless). If you need to change this, edit the `packages/gitea` package. You can find all of the `--set` configurations by looking at the `zarf-config.toml` in the root of the repository. diff --git a/examples/component-webhooks/package-lock.json b/examples/component-webhooks/package-lock.json index 1f55709f31..b50ee40013 100644 --- a/examples/component-webhooks/package-lock.json +++ b/examples/component-webhooks/package-lock.json @@ -8,7 +8,7 @@ "name": "example-webhook", "version": "0.0.1", "dependencies": { - "pepr": "^0.19.0" + "pepr": "^20.0.0" }, "engines": { "node": ">=18.0.0" @@ -1516,24 +1516,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -1546,25 +1528,6 @@ "node": ">=10.13.0" } }, - "node_modules/glob/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/globals": { "version": "13.23.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.23.0.tgz", @@ -1649,26 +1612,9 @@ } }, "node_modules/help-me": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/help-me/-/help-me-4.2.0.tgz", - "integrity": "sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA==", - "dependencies": { - "glob": "^8.0.0", - "readable-stream": "^3.6.0" - } - }, - "node_modules/help-me/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==" }, "node_modules/http-errors": { "version": "2.0.0", @@ -2360,16 +2306,16 @@ } }, "node_modules/pepr": { - "version": "0.19.0", - "resolved": "https://registry.npmjs.org/pepr/-/pepr-0.19.0.tgz", - "integrity": "sha512-DLhYZ/3J/jUNCMy9uQ/MXVXqXqBtI3IuVn89W2ESDX1j8uk6akTu7/hlm5cCjBrh71V7PFQuivFzTXiq1pqapw==", + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/pepr/-/pepr-20.0.0.tgz", + "integrity": "sha512-Pb84L0VkK5xtqBzZKjjD6xBJiMnypmvuqPpomL46hANeI0ks81rcWziFupQS4UiLnHcN2H32Il/bCYyzj5vFbQ==", "dependencies": { "@types/ramda": "0.29.9", "express": "4.18.2", "fast-json-patch": "3.1.1", "kubernetes-fluent-client": "1.9.0", - "pino": "8.16.2", - "pino-pretty": "10.2.3", + "pino": "8.17.1", + "pino-pretty": "10.3.0", "prom-client": "15.0.0", "ramda": "0.29.1" }, @@ -2405,9 +2351,9 @@ } }, "node_modules/pino": { - "version": "8.16.2", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.16.2.tgz", - "integrity": "sha512-2advCDGVEvkKu9TTVSa/kWW7Z3htI/sBKEZpqiHk6ive0i/7f5b1rsU8jn0aimxqfnSz5bj/nOYkwhBUn5xxvg==", + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-8.17.1.tgz", + "integrity": "sha512-YoN7/NJgnsJ+fkADZqjhRt96iepWBndQHeClmSBH0sQWCb8zGD74t00SK4eOtKFi/f8TUmQnfmgglEhd2kI1RQ==", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -2435,15 +2381,15 @@ } }, "node_modules/pino-pretty": { - "version": "10.2.3", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.3.tgz", - "integrity": "sha512-4jfIUc8TC1GPUfDyMSlW1STeORqkoxec71yhxIpLDQapUu8WOuoz2TTCoidrIssyz78LZC69whBMPIKCMbi3cw==", + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.3.0.tgz", + "integrity": "sha512-JthvQW289q3454mhM3/38wFYGWPiBMR28T3CpDNABzoTQOje9UKS7XCJQSnjWF9LQGQkGd8D7h0oq+qwiM3jFA==", "dependencies": { "colorette": "^2.0.7", "dateformat": "^4.6.3", "fast-copy": "^3.0.0", "fast-safe-stringify": "^2.1.1", - "help-me": "^4.0.1", + "help-me": "^5.0.0", "joycon": "^3.1.1", "minimist": "^1.2.6", "on-exit-leak-free": "^2.1.0", @@ -3262,11 +3208,6 @@ "requires-port": "^1.0.0" } }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" - }, "node_modules/utils-merge": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", diff --git a/examples/component-webhooks/package.json b/examples/component-webhooks/package.json index b96b43c649..8d91530a8b 100644 --- a/examples/component-webhooks/package.json +++ b/examples/component-webhooks/package.json @@ -25,6 +25,6 @@ "k3d-setup": "k3d cluster delete pepr-dev && k3d cluster create pepr-dev --k3s-arg '--debug@server:0'" }, "dependencies": { - "pepr": "^0.19.0" + "pepr": "^20.0.0" } } diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index 34f499d670..efd16ce6d9 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -31,7 +31,7 @@ components: # default: false # the initial value overrides the child component import: # The URL to the skeleton package containing this component's package definition - url: oci://🦄/dos-games:1.0.0-skeleton + url: oci://🦄/dos-games:1.0.0 # Example optional custom name to point to in the imported package (default is to use this component's name) name: baseline # Un'name'd Zarf primitives will be appended to the end of the primitive's list for that component. diff --git a/examples/git-data/zarf.yaml b/examples/git-data/zarf.yaml index ca61b0d252..f5262a93f1 100644 --- a/examples/git-data/zarf.yaml +++ b/examples/git-data/zarf.yaml @@ -6,7 +6,6 @@ metadata: components: - name: full-repo - required: true repos: # The following performs a full Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git @@ -14,7 +13,6 @@ components: - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test - name: specific-tag - required: true repos: # The following performs a tag Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@v0.0.1 @@ -24,7 +22,6 @@ components: - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 - name: specific-branch - required: true repos: # The following performs a branch Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons @@ -32,7 +29,6 @@ components: - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@refs/heads/dragons - name: specific-hash - required: true repos: # The following performs a SHA Git Repo Mirror with `go-git` (internal to Zarf) - https://github.com/defenseunicorns/zarf-public-test.git@01a23218923f24194133b5eb11268cf8d73ff1bb diff --git a/examples/package-flavors/zarf.yaml b/examples/package-flavors/zarf.yaml index 8650ce69e1..873b36f493 100644 --- a/examples/package-flavors/zarf.yaml +++ b/examples/package-flavors/zarf.yaml @@ -2,6 +2,7 @@ kind: ZarfPackageConfig metadata: name: package-flavors description: Simple example to show how to use the `only.flavor` key to build package variants. + version: 1.0.0 components: - name: image diff --git a/go.mod b/go.mod index 1aa2e7c054..50fd25a90a 100644 --- a/go.mod +++ b/go.mod @@ -5,17 +5,14 @@ go 1.21.5 // TODO (@AABRO): Pending merge into github.com/gojsonschema/gojsonschema (https://github.com/gojsonschema/gojsonschema/pull/5) replace github.com/xeipuuv/gojsonschema => github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 -// TODO (@WSTARR): Pending merge into github.com/anchore/syft (https://github.com/anchore/syft/pull/2411) -replace github.com/anchore/syft => github.com/defenseunicorns/syft v0.75.1-0.20231208231130-562ba667d3d1 - require ( cuelang.org/go v0.7.0 github.com/AlecAivazis/survey/v2 v2.3.7 github.com/Masterminds/semver/v3 v3.2.1 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b - github.com/anchore/clio v0.0.0-20231128152715-767f62261f13 - github.com/anchore/stereoscope v0.0.0-20231117203853-3610f4ef3e83 - github.com/anchore/syft v0.98.0 + github.com/anchore/clio v0.0.0-20240105134038-89e2fe85ce25 + github.com/anchore/stereoscope v0.0.0-20231220161148-590920dabc54 + github.com/anchore/syft v0.99.0 github.com/derailed/k9s v0.29.1 github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.7+incompatible @@ -23,7 +20,7 @@ require ( github.com/fatih/color v1.16.0 github.com/fluxcd/helm-controller/api v0.36.2 github.com/fluxcd/source-controller/api v1.2.1 - github.com/go-git/go-git/v5 v5.10.1 + github.com/go-git/go-git/v5 v5.11.0 github.com/go-logr/logr v1.3.0 github.com/goccy/go-yaml v1.11.2 github.com/gofrs/flock v0.8.1 @@ -47,9 +44,9 @@ require ( github.com/spf13/viper v1.18.1 github.com/stretchr/testify v1.8.4 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/crypto v0.16.0 + golang.org/x/crypto v0.17.0 golang.org/x/sync v0.5.0 - golang.org/x/term v0.15.0 + golang.org/x/term v0.16.0 helm.sh/helm/v3 v3.13.2 k8s.io/api v0.28.4 k8s.io/apimachinery v0.28.4 @@ -96,7 +93,7 @@ require ( github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect - github.com/CycloneDX/cyclonedx-go v0.7.2 // indirect + github.com/CycloneDX/cyclonedx-go v0.8.0 // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -129,7 +126,7 @@ require ( github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092 // indirect github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 // indirect - github.com/anchore/grype v0.73.4 // indirect + github.com/anchore/grype v0.73.5 // indirect github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 // indirect github.com/andybalholm/brotli v1.0.6 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect @@ -171,7 +168,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/charmbracelet/bubbles v0.16.1 // indirect - github.com/charmbracelet/bubbletea v0.24.2 // indirect + github.com/charmbracelet/bubbletea v0.25.0 // indirect github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/lipgloss v0.9.1 // indirect github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 // indirect @@ -181,7 +178,7 @@ require ( github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect - github.com/containerd/containerd v1.7.9 // indirect + github.com/containerd/containerd v1.7.11 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/log v0.1.0 // indirect @@ -219,6 +216,7 @@ require ( github.com/facebookincubator/nvdtools v0.1.5 // indirect github.com/fatih/camelcase v1.0.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect github.com/fluxcd/pkg/apis/kustomize v1.1.1 // indirect github.com/fluxcd/pkg/apis/meta v1.2.0 // indirect @@ -269,7 +267,7 @@ require ( github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gookit/color v1.5.4 // indirect @@ -397,7 +395,7 @@ require ( github.com/rubenv/sql-migrate v1.5.2 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect - github.com/saferwall/pe v1.4.7 // indirect + github.com/saferwall/pe v1.4.8 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect @@ -454,6 +452,7 @@ require ( go.mongodb.org/mongo-driver v1.12.1 // indirect go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect go.opentelemetry.io/otel/sdk v1.21.0 // indirect @@ -466,7 +465,7 @@ require ( golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.15.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.15.0 // indirect @@ -497,7 +496,7 @@ require ( modernc.org/libc v1.29.0 // indirect modernc.org/mathutil v1.6.0 // indirect modernc.org/memory v1.7.2 // indirect - modernc.org/sqlite v1.27.0 // indirect + modernc.org/sqlite v1.28.0 // indirect oras.land/oras-go v1.2.4 // indirect sigs.k8s.io/controller-runtime v0.16.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index b7ac061017..376522f5d2 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,8 @@ github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CycloneDX/cyclonedx-go v0.7.2 h1:kKQ0t1dPOlugSIYVOMiMtFqeXI2wp/f5DBIdfux8gnQ= -github.com/CycloneDX/cyclonedx-go v0.7.2/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= +github.com/CycloneDX/cyclonedx-go v0.8.0 h1:FyWVj6x6hoJrui5uRQdYZcSievw3Z32Z88uYzG/0D6M= +github.com/CycloneDX/cyclonedx-go v0.8.0/go.mod h1:K2bA+324+Og0X84fA8HhN2X066K7Bxz4rpMQ4ZhjtSk= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -364,8 +364,8 @@ github.com/aliyun/credentials-go v1.3.1 h1:uq/0v7kWrxmoLGpqjx7vtQ/s03f0zR//0br/x github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw= -github.com/anchore/clio v0.0.0-20231128152715-767f62261f13 h1:N7G209spgFIPoXDF1xfNwmH3yfY04iRW9btzoufiiCA= -github.com/anchore/clio v0.0.0-20231128152715-767f62261f13/go.mod h1:2uHfqEAL3w4ZXZQAG4x4rGAMZfiZqJkvjDLhH6Kuhro= +github.com/anchore/clio v0.0.0-20240105134038-89e2fe85ce25 h1:x3bHYneI+kQ7EHtBSeWVkzj8005/KPnzvPTI3oouHOQ= +github.com/anchore/clio v0.0.0-20240105134038-89e2fe85ce25/go.mod h1:5BGBD6yira9bK2N4Czz2rGnkdTq7oPAxBG1NZqfDoPU= github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b h1:L/djgY7ZbZ/38+wUtdkk398W3PIBJLkt1N8nU/7e47A= github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b/go.mod h1:TLcE0RE5+8oIx2/NPWem/dq1DeaMoC+fPEH7hoSzPLo= github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw= @@ -378,12 +378,14 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE+o2gozGEBoUMpX27lsku+xrMwlmBZJtbg= github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/grype v0.73.4 h1:j8HzRHbXLLZ6U2lmDDRFILd+VZtWbsfg/RYhatRZW9E= -github.com/anchore/grype v0.73.4/go.mod h1:5kJSAsHPoK47DsGZLHHArCfhHVGFGRkCfL2H87GdrdY= +github.com/anchore/grype v0.73.5 h1:1X81Snj5pGpl9ru7mQl1eYLX1Ek2ElfKhm9cwIgdCOw= +github.com/anchore/grype v0.73.5/go.mod h1:bdI7d2XeXQbmfbqql/Fqg+Lv2w4gO3nN3jfby/mBIcs= github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501 h1:AV7qjwMcM4r8wFhJq3jLRztew3ywIyPTRapl2T1s9o8= github.com/anchore/packageurl-go v0.1.1-0.20230104203445-02e0a6721501/go.mod h1:Blo6OgJNiYF41ufcgHKkbCKF2MDOMlrqhXv/ij6ocR4= -github.com/anchore/stereoscope v0.0.0-20231117203853-3610f4ef3e83 h1:mxGIOmj+asEm8LUkPTG3/v0hi27WIlDVjiEVsUB9eqY= -github.com/anchore/stereoscope v0.0.0-20231117203853-3610f4ef3e83/go.mod h1:GKAnytSVV1hoqB5r5Gd9M5Ph3Rzqq0zPdEJesewjC2w= +github.com/anchore/stereoscope v0.0.0-20231220161148-590920dabc54 h1:i2YK5QEs9H2YB3B2zv+AGR44ves0nmAGOD07lMphH14= +github.com/anchore/stereoscope v0.0.0-20231220161148-590920dabc54/go.mod h1:IylG7ofLoUKHwS1XDF6rPhOmaE3GgpAgsMdvvYfooTU= +github.com/anchore/syft v0.99.0 h1:oqycIA7XfHCB09meroN7eY2RWTGUZIdtWsMQL2HlPvw= +github.com/anchore/syft v0.99.0/go.mod h1:tGZGyDxB2z/yu+x266+b67fMenGKCrUvSNVKED1euuo= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= @@ -507,8 +509,8 @@ github.com/chai2010/gettext-go v1.0.2 h1:1Lwwip6Q2QGsAdl/ZKPCwTe9fe0CjlUbqj5bFNS github.com/chai2010/gettext-go v1.0.2/go.mod h1:y+wnP2cHYaVj19NZhYKAwEMH2CI1gNHeQQ+5AjwawxA= github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= -github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= -github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= +github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= +github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= @@ -551,8 +553,8 @@ github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHq github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= -github.com/containerd/containerd v1.7.9 h1:KOhK01szQbM80YfW1H6RZKh85PHGqY/9OcEZ35Je8sc= -github.com/containerd/containerd v1.7.9/go.mod h1:0/W44LWEYfSHoxBtsHIiNU/duEkgpMokemafHVCpq9Y= +github.com/containerd/containerd v1.7.11 h1:lfGKw3eU35sjV0aG2eYZTiwFEY1pCzxdzicHP3SZILw= +github.com/containerd/containerd v1.7.11/go.mod h1:5UluHxHTX2rdvYuZ5OJTC5m/KJNs0Zs9wVoJm9zf5ZE= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= @@ -593,8 +595,6 @@ github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gwevOZ0fxT2nzM9hrtdPbsiOHjFqDRIYMzJHba3/G6Q= github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= -github.com/defenseunicorns/syft v0.75.1-0.20231208231130-562ba667d3d1 h1:FIadwOhSeyktN20XyLmDNUCMyI3T+ZGCI/w7hvWk2xE= -github.com/defenseunicorns/syft v0.75.1-0.20231208231130-562ba667d3d1/go.mod h1:iD9FSCgyXpX+5Ze0BCzI7fngzT97kzaCqGnCdcQyF6E= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= @@ -749,8 +749,8 @@ github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+ github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.10.1 h1:tu8/D8i+TWxgKpzQ3Vc43e+kkhXqtsZCKI/egajKnxk= -github.com/go-git/go-git/v5 v5.10.1/go.mod h1:uEuHjxkHap8kAl//V5F/nNWwqIYtP/402ddd05mp0wg= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -974,8 +974,8 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= @@ -1492,8 +1492,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/saferwall/pe v1.4.7 h1:A+G3DxX49paJ5OsxBfHKskhyDtmTjShlDmBd81IsHlQ= -github.com/saferwall/pe v1.4.7/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= +github.com/saferwall/pe v1.4.8 h1:ey/L8FGBMrJ1Xh+Rltj1MAFPZ4LOQYGJqNa5B1Na6B0= +github.com/saferwall/pe v1.4.8/go.mod h1:SNzv3cdgk8SBI0UwHfyTcdjawfdnN+nbydnEL7GZ25s= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= @@ -1796,8 +1796,8 @@ golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2Uz golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -2059,8 +2059,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2073,8 +2073,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2484,8 +2484,8 @@ modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= -modernc.org/sqlite v1.27.0 h1:MpKAHoyYB7xqcwnUwkuD+npwEa0fojF0B5QRbN+auJ8= -modernc.org/sqlite v1.27.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= +modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ= +modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= oras.land/oras-go/v2 v2.3.1 h1:lUC6q8RkeRReANEERLfH86iwGn55lbSWP20egdFHVec= diff --git a/hack/lint_all_zarf_packages.sh b/hack/lint_all_zarf_packages.sh new file mode 100755 index 0000000000..5f41d3d887 --- /dev/null +++ b/hack/lint_all_zarf_packages.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +ZARF_BIN=$1 +LINT_SRC_TEST=$2 +SCRIPT=$(realpath "$0") +SCRIPTPATH=$(dirname "$SCRIPT") +cd "$SCRIPTPATH" || exit +cd .. +find "." -type f -name 'zarf.yaml' | while read -r yaml_file; do + dir=$(dirname "$yaml_file") + if [[ "$dir" == *src/test/* ]] && [ "$LINT_SRC_TEST" != true ]; then + continue + fi + echo "Running 'zarf prepare lint' in directory: $dir" + $ZARF_BIN prepare lint "$dir" + echo "---" +done diff --git a/packages/gitea/gitea-values.yaml b/packages/gitea/gitea-values.yaml index d8b4aa8c22..4a780198b3 100644 --- a/packages/gitea/gitea-values.yaml +++ b/packages/gitea/gitea-values.yaml @@ -1,11 +1,12 @@ persistence: storageClass: "###ZARF_STORAGE_CLASS###" - existingClaim: "###ZARF_VAR_GIT_SERVER_EXISTING_PVC###" + claimName: "###ZARF_VAR_GIT_SERVER_EXISTING_PVC###" size: "###ZARF_VAR_GIT_SERVER_PVC_SIZE###" accessModes: - "###ZARF_VAR_GIT_SERVER_PVC_ACCESS_MODE###" + create: ###ZARF_VAR_GIT_SERVER_CREATE_PVC### -replicaCount: "###ZARF_VAR_GIT_SERVER_REPLICA_COUNT###" +replicaCount: ###ZARF_VAR_GIT_SERVER_REPLICA_COUNT### gitea: admin: @@ -29,6 +30,12 @@ gitea: repository: ENABLE_PUSH_CREATE_USER: true FORCE_PRIVATE: true + session: + PROVIDER: memory + cache: + ADAPTER: memory + queue: + TYPE: level resources: requests: cpu: "###ZARF_VAR_GIT_SERVER_CPU_REQ###" @@ -37,13 +44,12 @@ resources: cpu: "###ZARF_VAR_GIT_SERVER_CPU_LIMIT###" memory: "###ZARF_VAR_GIT_SERVER_MEM_LIMIT###" -memcached: - enabled: false +image: + fullOverride: "###ZARF_CONST_GITEA_IMAGE###" + rootless: true -postgresql: +postgresql-ha: enabled: false -image: - repository: "###ZARF_CONST_GITEA_IMAGE###" - tag: "###ZARF_CONST_GITEA_SERVER_VERSION###" - rootless: true +redis-cluster: + enabled: false diff --git a/packages/gitea/zarf.yaml b/packages/gitea/zarf.yaml index 62ab2b0497..8fb37939eb 100644 --- a/packages/gitea/zarf.yaml +++ b/packages/gitea/zarf.yaml @@ -5,7 +5,7 @@ metadata: variables: - name: GIT_SERVER_EXISTING_PVC description: "Optional: Use an existing PVC for the git server instead of creating a new one. If this is set, the GIT_SERVER_PVC_SIZE variable will be ignored." - default: "" + default: "data-zarf-gitea-0" - name: GIT_SERVER_PVC_SIZE description: The size of the persistent volume claim for the git server @@ -42,8 +42,6 @@ variables: constants: - name: GITEA_IMAGE value: "###ZARF_PKG_TMPL_GITEA_IMAGE###" - - name: GITEA_SERVER_VERSION - value: "###ZARF_PKG_TMPL_GITEA_SERVER_VERSION###" components: - name: git-server @@ -51,7 +49,7 @@ components: Deploys Gitea to provide git repositories for Kubernetes configurations. Required for GitOps deployments if no other git server is available. images: - - "###ZARF_PKG_TMPL_GITEA_IMAGE###:###ZARF_PKG_TMPL_GITEA_SERVER_VERSION###-rootless" + - "###ZARF_PKG_TMPL_GITEA_IMAGE###" manifests: - name: git-connect namespace: zarf @@ -61,13 +59,24 @@ components: - name: gitea releaseName: zarf-gitea url: https://dl.gitea.io/charts - version: 8.3.0 + version: 10.0.0 namespace: zarf valuesFiles: - gitea-values.yaml actions: onDeploy: + before: + - cmd: ./zarf internal update-gitea-pvc --no-progress + setVariables: + - name: GIT_SERVER_CREATE_PVC + mute: true after: + - wait: + cluster: + kind: pod + namespace: zarf + name: app=gitea + condition: Ready - cmd: ./zarf internal create-read-only-gitea-user --no-progress maxRetries: 3 maxTotalSeconds: 60 @@ -76,9 +85,6 @@ components: maxRetries: 3 maxTotalSeconds: 60 description: Create an artifact registry token - - wait: - cluster: - kind: pod - namespace: zarf - name: app=gitea - condition: Ready + + onFailure: + - cmd: ./zarf internal update-gitea-pvc --rollback --no-progress diff --git a/src/cmd/common/utils.go b/src/cmd/common/utils.go new file mode 100644 index 0000000000..01a6b104d6 --- /dev/null +++ b/src/cmd/common/utils.go @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package common handles command configuration across all commands +package common + +import ( + "github.com/defenseunicorns/zarf/src/types" +) + +// SetBaseDirectory sets base directory on package config when given in args +func SetBaseDirectory(args []string, pkgConfig *types.PackagerConfig) { + if len(args) > 0 { + pkgConfig.CreateOpts.BaseDir = args[0] + } else { + pkgConfig.CreateOpts.BaseDir = "." + } +} diff --git a/src/cmd/common/viper.go b/src/cmd/common/viper.go index 4b040179b9..34d6d0888b 100644 --- a/src/cmd/common/viper.go +++ b/src/cmd/common/viper.go @@ -93,6 +93,10 @@ const ( // Package pull config keys VPkgPullOutputDir = "package.pull.output_directory" + + // Dev deploy config keys + + VDevDeployNoYolo = "dev.deploy.no_yolo" ) var ( diff --git a/src/cmd/connect.go b/src/cmd/connect.go index 131faa0f7c..73fab7665a 100644 --- a/src/cmd/connect.go +++ b/src/cmd/connect.go @@ -74,9 +74,13 @@ var ( signal.Notify(interruptChan, os.Interrupt, syscall.SIGTERM) exec.SuppressGlobalInterrupt = true - // Wait for the interrupt signal. - <-interruptChan - spinner.Successf(lang.CmdConnectTunnelClosed, url) + // Wait for the interrupt signal or an error. + select { + case err = <-tunnel.ErrChan(): + spinner.Fatalf(err, lang.CmdConnectErrService, err.Error()) + case <-interruptChan: + spinner.Successf(lang.CmdConnectTunnelClosed, url) + } os.Exit(0) }, } diff --git a/src/cmd/dev.go b/src/cmd/dev.go new file mode 100644 index 0000000000..63426692ea --- /dev/null +++ b/src/cmd/dev.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package cmd contains the CLI commands for Zarf. +package cmd + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/AlecAivazis/survey/v2" + "github.com/defenseunicorns/zarf/src/cmd/common" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/packager" + "github.com/defenseunicorns/zarf/src/pkg/packager/lint" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/mholt/archiver/v3" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +var extractPath string + +var devCmd = &cobra.Command{ + Use: "dev", + Aliases: []string{"prepare", "prep"}, + Short: lang.CmdDevShort, +} + +var devDeployCmd = &cobra.Command{ + Use: "deploy", + Args: cobra.MaximumNArgs(1), + Short: lang.CmdDevDeployShort, + Long: lang.CmdDevDeployLong, + Run: func(cmd *cobra.Command, args []string) { + common.SetBaseDirectory(args, &pkgConfig) + + v := common.GetViper() + pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) + + pkgConfig.PkgOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgDeploySet), pkgConfig.PkgOpts.SetVariables, strings.ToUpper) + + // Configure the packager + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + + // Create the package + if err := pkgClient.DevDeploy(); err != nil { + message.Fatalf(err, lang.CmdDevDeployErr, err.Error()) + } + }, +} + +var devTransformGitLinksCmd = &cobra.Command{ + Use: "patch-git HOST FILE", + Aliases: []string{"p"}, + Short: lang.CmdDevPatchGitShort, + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + host, fileName := args[0], args[1] + + // Read the contents of the given file + content, err := os.ReadFile(fileName) + if err != nil { + message.Fatalf(err, lang.CmdDevPatchGitFileReadErr, fileName) + } + + pkgConfig.InitOpts.GitServer.Address = host + + // Perform git url transformation via regex + text := string(content) + processedText := transform.MutateGitURLsInText(message.Warnf, pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) + + // Print the differences + message.PrintDiff(text, processedText) + + // Ask the user before this destructive action + confirm := false + prompt := &survey.Confirm{ + Message: fmt.Sprintf(lang.CmdDevPatchGitOverwritePrompt, fileName), + } + if err := survey.AskOne(prompt, &confirm); err != nil { + message.Fatalf(nil, lang.CmdDevPatchGitOverwriteErr, err.Error()) + } + + if confirm { + // Overwrite the file + err = os.WriteFile(fileName, []byte(processedText), 0640) + if err != nil { + message.Fatal(err, lang.CmdDevPatchGitFileWriteErr) + } + } + + }, +} + +var devSha256SumCmd = &cobra.Command{ + Use: "sha256sum { FILE | URL }", + Aliases: []string{"s"}, + Short: lang.CmdDevSha256sumShort, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + fileName := args[0] + + var tmp string + var data io.ReadCloser + var err error + + if helpers.IsURL(fileName) { + message.Warn(lang.CmdDevSha256sumRemoteWarning) + + fileBase, err := helpers.ExtractBasePathFromURL(fileName) + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } + + if fileBase == "" { + fileBase = "sha-file" + } + + tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } + + downloadPath := filepath.Join(tmp, fileBase) + err = utils.DownloadToFile(fileName, downloadPath, "") + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } + + fileName = downloadPath + + defer os.RemoveAll(tmp) + } + + if extractPath != "" { + if tmp == "" { + tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } + defer os.RemoveAll(tmp) + } + + extractedFile := filepath.Join(tmp, extractPath) + + err = archiver.Extract(fileName, extractPath, tmp) + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } + + fileName = extractedFile + } + + data, err = os.Open(fileName) + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } + defer data.Close() + + var hash string + hash, err = helpers.GetSHA256Hash(data) + if err != nil { + message.Fatalf(err, lang.CmdDevSha256sumHashErr, err.Error()) + } else { + fmt.Println(hash) + } + }, +} + +var devFindImagesCmd = &cobra.Command{ + Use: "find-images [ PACKAGE ]", + Aliases: []string{"f"}, + Args: cobra.MaximumNArgs(1), + Short: lang.CmdDevFindImagesShort, + Long: lang.CmdDevFindImagesLong, + Run: func(cmd *cobra.Command, args []string) { + // If a directory was provided, use that as the base directory + common.SetBaseDirectory(args, &pkgConfig) + + // Ensure uppercase keys from viper + v := common.GetViper() + pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) + + // Configure the packager + pkgClient := packager.NewOrDie(&pkgConfig) + defer pkgClient.ClearTempPaths() + + // Find all the images the package might need + if _, err := pkgClient.FindImages(); err != nil { + message.Fatalf(err, lang.CmdDevFindImagesErr, err.Error()) + } + }, +} + +var devGenConfigFileCmd = &cobra.Command{ + Use: "generate-config [ FILENAME ]", + Aliases: []string{"gc"}, + Args: cobra.MaximumNArgs(1), + Short: lang.CmdDevGenerateConfigShort, + Long: lang.CmdDevGenerateConfigLong, + Run: func(cmd *cobra.Command, args []string) { + fileName := "zarf-config.toml" + + // If a filename was provided, use that + if len(args) > 0 { + fileName = args[0] + } + + v := common.GetViper() + if err := v.SafeWriteConfigAs(fileName); err != nil { + message.Fatalf(err, lang.CmdDevGenerateConfigErr, fileName) + } + }, +} + +var devLintCmd = &cobra.Command{ + Use: "lint [ DIRECTORY ]", + Args: cobra.MaximumNArgs(1), + Aliases: []string{"l"}, + Short: lang.CmdDevLintShort, + Long: lang.CmdDevLintLong, + Run: func(cmd *cobra.Command, args []string) { + common.SetBaseDirectory(args, &pkgConfig) + v := common.GetViper() + pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( + v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) + validator, err := lint.Validate(pkgConfig.CreateOpts) + if err != nil { + message.Fatal(err, err.Error()) + } + validator.DisplayFormattedMessage() + if !validator.IsSuccess() { + os.Exit(1) + } + }, +} + +func init() { + v := common.GetViper() + rootCmd.AddCommand(devCmd) + + devCmd.AddCommand(devDeployCmd) + devCmd.AddCommand(devTransformGitLinksCmd) + devCmd.AddCommand(devSha256SumCmd) + devCmd.AddCommand(devFindImagesCmd) + devCmd.AddCommand(devGenConfigFileCmd) + devCmd.AddCommand(devLintCmd) + + bindDevDeployFlags(v) + + devSha256SumCmd.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdDevFlagExtractPath) + + devFindImagesCmd.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdDevFlagRepoChartPath) + // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables + devFindImagesCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdDevFlagSet) + // allow for the override of the default helm KubeVersion + devFindImagesCmd.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdDevFlagKubeVersion) + + devLintCmd.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) + devLintCmd.Flags().StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor) + devTransformGitLinksCmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdDevFlagGitAccount) +} + +func bindDevDeployFlags(v *viper.Viper) { + devDeployFlags := devDeployCmd.Flags() + + devDeployFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "create-set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPackageCreateFlagSet) + devDeployFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(common.VPkgCreateRegistryOverride), lang.CmdPackageCreateFlagRegistryOverride) + devDeployFlags.StringVarP(&pkgConfig.CreateOpts.Flavor, "flavor", "f", v.GetString(common.VPkgCreateFlavor), lang.CmdPackageCreateFlagFlavor) + + devDeployFlags.StringToStringVar(&pkgConfig.PkgOpts.SetVariables, "deploy-set", v.GetStringMapString(common.VPkgDeploySet), lang.CmdPackageDeployFlagSet) + + devDeployFlags.StringVar(&pkgConfig.PkgOpts.OptionalComponents, "components", v.GetString(common.VPkgDeployComponents), lang.CmdPackageDeployFlagComponents) + + devDeployFlags.BoolVar(&pkgConfig.CreateOpts.NoYOLO, "no-yolo", v.GetBool(common.VDevDeployNoYolo), lang.CmdDevDeployFlagNoYolo) +} diff --git a/src/cmd/initialize.go b/src/cmd/initialize.go index ea15dd7457..799896b066 100644 --- a/src/cmd/initialize.go +++ b/src/cmd/initialize.go @@ -119,7 +119,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { } var confirmDownload bool - url := oci.GetInitPackageURL(config.GetArch(), config.CLIVersion) + url := oci.GetInitPackageURL(config.CLIVersion) // Give the user the choice to download the init-package and note that this does require an internet connection message.Question(fmt.Sprintf(lang.CmdInitPullAsk, url)) @@ -138,7 +138,7 @@ func downloadInitPackage(cacheDirectory string) (string, error) { // If the user wants to download the init-package, download it if confirmDownload { - remote, err := oci.NewOrasRemote(url) + remote, err := oci.NewOrasRemote(url, oci.WithArch(config.GetArch())) if err != nil { return "", err } diff --git a/src/cmd/internal.go b/src/cmd/internal.go index 3dd40230f9..a10067398c 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -23,11 +23,14 @@ import ( "github.com/spf13/pflag" ) +var ( + rollback bool +) + var internalCmd = &cobra.Command{ - Use: "internal", - Aliases: []string{"dev"}, - Hidden: true, - Short: lang.CmdInternalShort, + Use: "internal", + Hidden: true, + Short: lang.CmdInternalShort, } var agentCmd = &cobra.Command{ @@ -197,6 +200,22 @@ var createPackageRegistryToken = &cobra.Command{ }, } +var updateGiteaPVC = &cobra.Command{ + Use: "update-gitea-pvc", + Short: lang.CmdInternalUpdateGiteaPVCShort, + Long: lang.CmdInternalUpdateGiteaPVCLong, + Run: func(cmd *cobra.Command, args []string) { + + // There is a possibility that the pvc does not yet exist and Gitea helm chart should create it + helmShouldCreate, err := git.UpdateGiteaPVC(rollback) + if err != nil { + message.WarnErr(err, lang.CmdInternalUpdateGiteaPVCErr) + } + + fmt.Print(helmShouldCreate) + }, +} + var isValidHostname = &cobra.Command{ Use: "is-valid-hostname", Short: lang.CmdInternalIsValidHostnameShort, @@ -230,8 +249,11 @@ func init() { internalCmd.AddCommand(genTypesSchemaCmd) internalCmd.AddCommand(createReadOnlyGiteaUser) internalCmd.AddCommand(createPackageRegistryToken) + internalCmd.AddCommand(updateGiteaPVC) internalCmd.AddCommand(isValidHostname) internalCmd.AddCommand(computeCrc32) + + updateGiteaPVC.Flags().BoolVarP(&rollback, "rollback", "r", false, lang.CmdInternalFlagUpdateGiteaPVCRollback) } func addHiddenDummyFlag(cmd *cobra.Command, flagDummy string) { diff --git a/src/cmd/package.go b/src/cmd/package.go index 42fc9fbc34..1ccd3e3061 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -6,7 +6,6 @@ package cmd import ( "fmt" - "os" "path/filepath" "regexp" "strings" @@ -41,17 +40,7 @@ var packageCreateCmd = &cobra.Command{ Short: lang.CmdPackageCreateShort, Long: lang.CmdPackageCreateLong, Run: func(cmd *cobra.Command, args []string) { - - // If a directory was provided, use that as the base directory - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - var err error - pkgConfig.CreateOpts.BaseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPackageCreateErr, err.Error()) - } - } + common.SetBaseDirectory(args, &pkgConfig) var isCleanPathRegex = regexp.MustCompile(`^[a-zA-Z0-9\_\-\/\.\~\\:]+$`) if !isCleanPathRegex.MatchString(config.CommonOptions.CachePath) { @@ -226,6 +215,7 @@ var packagePublishCmd = &cobra.Command{ if utils.IsDir(pkgConfig.PkgOpts.PackageSource) { pkgConfig.CreateOpts.BaseDir = pkgConfig.PkgOpts.PackageSource + pkgConfig.CreateOpts.IsSkeleton = true } pkgConfig.PublishOpts.PackageDestination = ref.String() diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go deleted file mode 100644 index fcd5e113c8..0000000000 --- a/src/cmd/prepare.go +++ /dev/null @@ -1,256 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package cmd contains the CLI commands for Zarf. -package cmd - -import ( - "fmt" - "io" - "os" - "path/filepath" - "strings" - - "github.com/AlecAivazis/survey/v2" - "github.com/defenseunicorns/zarf/src/cmd/common" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/packager" - "github.com/defenseunicorns/zarf/src/pkg/packager/lint" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/mholt/archiver/v3" - "github.com/spf13/cobra" -) - -var extractPath string - -var prepareCmd = &cobra.Command{ - Use: "prepare", - Aliases: []string{"prep"}, - Short: lang.CmdPrepareShort, -} - -var prepareTransformGitLinks = &cobra.Command{ - Use: "patch-git HOST FILE", - Aliases: []string{"p"}, - Short: lang.CmdPreparePatchGitShort, - Args: cobra.ExactArgs(2), - Run: func(cmd *cobra.Command, args []string) { - host, fileName := args[0], args[1] - - // Read the contents of the given file - content, err := os.ReadFile(fileName) - if err != nil { - message.Fatalf(err, lang.CmdPreparePatchGitFileReadErr, fileName) - } - - pkgConfig.InitOpts.GitServer.Address = host - - // Perform git url transformation via regex - text := string(content) - processedText := transform.MutateGitURLsInText(message.Warnf, pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) - - // Print the differences - message.PrintDiff(text, processedText) - - // Ask the user before this destructive action - confirm := false - prompt := &survey.Confirm{ - Message: fmt.Sprintf(lang.CmdPreparePatchGitOverwritePrompt, fileName), - } - if err := survey.AskOne(prompt, &confirm); err != nil { - message.Fatalf(nil, lang.CmdPreparePatchGitOverwriteErr, err.Error()) - } - - if confirm { - // Overwrite the file - err = os.WriteFile(fileName, []byte(processedText), 0640) - if err != nil { - message.Fatal(err, lang.CmdPreparePatchGitFileWriteErr) - } - } - - }, -} - -var prepareComputeFileSha256sum = &cobra.Command{ - Use: "sha256sum { FILE | URL }", - Aliases: []string{"s"}, - Short: lang.CmdPrepareSha256sumShort, - Args: cobra.ExactArgs(1), - Run: func(cmd *cobra.Command, args []string) { - fileName := args[0] - - var tmp string - var data io.ReadCloser - var err error - - if helpers.IsURL(fileName) { - message.Warn(lang.CmdPrepareSha256sumRemoteWarning) - - fileBase, err := helpers.ExtractBasePathFromURL(fileName) - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } - - if fileBase == "" { - fileBase = "sha-file" - } - - tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } - - downloadPath := filepath.Join(tmp, fileBase) - err = utils.DownloadToFile(fileName, downloadPath, "") - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } - - fileName = downloadPath - - defer os.RemoveAll(tmp) - } - - if extractPath != "" { - if tmp == "" { - tmp, err = utils.MakeTempDir(config.CommonOptions.TempDirectory) - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } - defer os.RemoveAll(tmp) - } - - extractedFile := filepath.Join(tmp, extractPath) - - err = archiver.Extract(fileName, extractPath, tmp) - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } - - fileName = extractedFile - } - - data, err = os.Open(fileName) - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } - defer data.Close() - - var hash string - hash, err = helpers.GetSHA256Hash(data) - if err != nil { - message.Fatalf(err, lang.CmdPrepareSha256sumHashErr, err.Error()) - } else { - fmt.Println(hash) - } - }, -} - -var prepareFindImages = &cobra.Command{ - Use: "find-images [ PACKAGE ]", - Aliases: []string{"f"}, - Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareFindImagesShort, - Long: lang.CmdPrepareFindImagesLong, - Run: func(cmd *cobra.Command, args []string) { - // If a directory was provided, use that as the base directory - if len(args) > 0 { - pkgConfig.CreateOpts.BaseDir = args[0] - } else { - cwd, err := os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) - } - pkgConfig.CreateOpts.BaseDir = cwd - } - - // Ensure uppercase keys from viper - v := common.GetViper() - pkgConfig.CreateOpts.SetVariables = helpers.TransformAndMergeMap( - v.GetStringMapString(common.VPkgCreateSet), pkgConfig.CreateOpts.SetVariables, strings.ToUpper) - - // Configure the packager - pkgClient := packager.NewOrDie(&pkgConfig) - defer pkgClient.ClearTempPaths() - - // Find all the images the package might need - if _, err := pkgClient.FindImages(); err != nil { - message.Fatalf(err, lang.CmdPrepareFindImagesErr, err.Error()) - } - }, -} - -var prepareGenerateConfigFile = &cobra.Command{ - Use: "generate-config [ FILENAME ]", - Aliases: []string{"gc"}, - Args: cobra.MaximumNArgs(1), - Short: lang.CmdPrepareGenerateConfigShort, - Long: lang.CmdPrepareGenerateConfigLong, - Run: func(cmd *cobra.Command, args []string) { - fileName := "zarf-config.toml" - - // If a filename was provided, use that - if len(args) > 0 { - fileName = args[0] - } - - v := common.GetViper() - if err := v.SafeWriteConfigAs(fileName); err != nil { - message.Fatalf(err, lang.CmdPrepareGenerateConfigErr, fileName) - } - }, -} - -var lintCmd = &cobra.Command{ - Use: "lint [ DIRECTORY ]", - Args: cobra.MaximumNArgs(1), - Aliases: []string{"l"}, - Short: lang.CmdPrepareLintShort, - Long: lang.CmdPrepareLintLong, - Run: func(cmd *cobra.Command, args []string) { - baseDir := "" - if len(args) > 0 { - baseDir = args[0] - } else { - var err error - baseDir, err = os.Getwd() - if err != nil { - message.Fatalf(err, lang.CmdPrepareLintErr, err.Error()) - } - } - validator, err := lint.ValidateZarfSchema(baseDir) - if err != nil { - message.Fatal(err, err.Error()) - } - validator.DisplayFormattedMessage() - if !validator.IsSuccess() { - os.Exit(1) - } - }, -} - -func init() { - v := common.InitViper() - - rootCmd.AddCommand(prepareCmd) - prepareCmd.AddCommand(prepareTransformGitLinks) - prepareCmd.AddCommand(prepareComputeFileSha256sum) - prepareCmd.AddCommand(prepareFindImages) - prepareCmd.AddCommand(prepareGenerateConfigFile) - prepareCmd.AddCommand(lintCmd) - - prepareComputeFileSha256sum.Flags().StringVarP(&extractPath, "extract-path", "e", "", lang.CmdPrepareFlagExtractPath) - - prepareFindImages.Flags().StringVarP(&pkgConfig.FindImagesOpts.RepoHelmChartPath, "repo-chart-path", "p", "", lang.CmdPrepareFlagRepoChartPath) - // use the package create config for this and reset it here to avoid overwriting the config.CreateOptions.SetVariables - prepareFindImages.Flags().StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(common.VPkgCreateSet), lang.CmdPrepareFlagSet) - // allow for the override of the default helm KubeVersion - prepareFindImages.Flags().StringVar(&pkgConfig.FindImagesOpts.KubeVersionOverride, "kube-version", "", lang.CmdPrepareFlagKubeVersion) - - prepareTransformGitLinks.Flags().StringVar(&pkgConfig.InitOpts.GitServer.PushUsername, "git-account", config.ZarfGitPushUser, lang.CmdPrepareFlagGitAccount) -} diff --git a/src/cmd/tools/archiver.go b/src/cmd/tools/archiver.go index aafcba6747..e392b8c3ad 100644 --- a/src/cmd/tools/archiver.go +++ b/src/cmd/tools/archiver.go @@ -53,6 +53,9 @@ var archiverDecompressCmd = &cobra.Command{ if unarchiveAll { err := filepath.Walk(destinationPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } if strings.HasSuffix(path, ".tar") { dst := filepath.Join(strings.TrimSuffix(path, ".tar"), "..") // Unpack sboms.tar differently since it has a different folder structure than components @@ -71,7 +74,7 @@ var archiverDecompressCmd = &cobra.Command{ return nil }) if err != nil { - message.Fatalf(err, lang.CmdToolsArchiverUnarchiveAllErr) + message.Fatalf(err, lang.CmdToolsArchiverUnarchiveAllErr, err.Error()) } } }, diff --git a/src/cmd/tools/crane.go b/src/cmd/tools/crane.go index fb7274eeff..80030fd361 100644 --- a/src/cmd/tools/crane.go +++ b/src/cmd/tools/crane.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" + "github.com/defenseunicorns/zarf/src/types" craneCmd "github.com/google/go-containerregistry/cmd/crane/cmd" "github.com/google/go-containerregistry/pkg/crane" "github.com/google/go-containerregistry/pkg/logs" @@ -132,15 +133,16 @@ func zarfCraneCatalog(cranePlatformOptions *[]crane.Option) *cobra.Command { return err } + // Add the correct authentication to the crane command options + authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PullUsername, zarfState.RegistryInfo.PullPassword) + *cranePlatformOptions = append(*cranePlatformOptions, authOption) + if tunnel != nil { message.Notef(lang.CmdToolsRegistryTunnel, registryEndpoint, zarfState.RegistryInfo.Address) defer tunnel.Close() + return tunnel.Wrap(func() error { return originalCatalogFn(cmd, []string{registryEndpoint}) }) } - // Add the correct authentication to the crane command options - authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PullUsername, zarfState.RegistryInfo.PullPassword) - *cranePlatformOptions = append(*cranePlatformOptions, authOption) - return originalCatalogFn(cmd, []string{registryEndpoint}) } @@ -186,6 +188,10 @@ func zarfCraneInternalWrapper(commandToWrap func(*[]crane.Option) *cobra.Command return err } + // Add the correct authentication to the crane command options + authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PushUsername, zarfState.RegistryInfo.PushPassword) + *cranePlatformOptions = append(*cranePlatformOptions, authOption) + if tunnel != nil { message.Notef(lang.CmdToolsRegistryTunnel, tunnel.Endpoint(), zarfState.RegistryInfo.Address) @@ -194,12 +200,9 @@ func zarfCraneInternalWrapper(commandToWrap func(*[]crane.Option) *cobra.Command givenAddress := fmt.Sprintf("%s/", zarfState.RegistryInfo.Address) tunnelAddress := fmt.Sprintf("%s/", tunnel.Endpoint()) args[imageNameArgumentIndex] = strings.Replace(args[imageNameArgumentIndex], givenAddress, tunnelAddress, 1) + return tunnel.Wrap(func() error { return originalListFn(cmd, args) }) } - // Add the correct authentication to the crane command options - authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PushUsername, zarfState.RegistryInfo.PushPassword) - *cranePlatformOptions = append(*cranePlatformOptions, authOption) - return originalListFn(cmd, args) } @@ -234,8 +237,13 @@ func pruneImages(_ *cobra.Command, _ []string) error { if tunnel != nil { message.Notef(lang.CmdToolsRegistryTunnel, registryEndpoint, zarfState.RegistryInfo.Address) defer tunnel.Close() + return tunnel.Wrap(func() error { return doPruneImagesForPackages(zarfState, zarfPackages, registryEndpoint) }) } + return doPruneImagesForPackages(zarfState, zarfPackages, registryEndpoint) +} + +func doPruneImagesForPackages(zarfState *types.ZarfState, zarfPackages []types.DeployedPackage, registryEndpoint string) error { authOption := config.GetCraneAuthOption(zarfState.RegistryInfo.PushUsername, zarfState.RegistryInfo.PushPassword) // Determine which image digests are currently used by Zarf packages diff --git a/src/cmd/tools/zarf.go b/src/cmd/tools/zarf.go index bbd8020e07..d835674ed6 100644 --- a/src/cmd/tools/zarf.go +++ b/src/cmd/tools/zarf.go @@ -88,8 +88,10 @@ var updateCredsCmd = &cobra.Command{ // If no distro the zarf secret did not load properly message.Fatalf(nil, lang.ErrLoadState) } - - newState := c.MergeZarfState(oldState, updateCredsInitOpts, args) + var newState *types.ZarfState + if newState, err = c.MergeZarfState(oldState, updateCredsInitOpts, args); err != nil { + message.Fatal(err, lang.CmdToolsUpdateCredsUnableUpdateCreds) + } message.PrintCredentialUpdates(oldState, newState, args) @@ -144,7 +146,8 @@ var updateCredsCmd = &cobra.Command{ } } if slices.Contains(args, message.GitKey) && newState.GitServer.InternalServer { - err = h.UpdateZarfGiteaValues() + g := git.New(newState.GitServer) + err = g.UpdateZarfGiteaUsers(oldState) if err != nil { // Warn if we couldn't actually update the git server (it might not be installed and we should try to continue) message.Warnf(lang.CmdToolsUpdateCredsUnableUpdateGit, err.Error()) @@ -178,9 +181,9 @@ var downloadInitCmd = &cobra.Command{ Use: "download-init", Short: lang.CmdToolsDownloadInitShort, Run: func(cmd *cobra.Command, args []string) { - url := oci.GetInitPackageURL(config.GetArch(), config.CLIVersion) + url := oci.GetInitPackageURL(config.CLIVersion) - remote, err := oci.NewOrasRemote(url) + remote, err := oci.NewOrasRemote(url, oci.WithArch(config.GetArch())) if err != nil { message.Fatalf(err, lang.CmdToolsDownloadInitErr, err.Error()) } diff --git a/src/config/lang/english.go b/src/config/lang/english.go index d14bf0b23d..6a666474d7 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -15,20 +15,26 @@ import "errors" // Debug messages will not be a part of the language strings since they are not intended to be user facing // Include sprintf formatting directives in the string if needed. const ( - ErrLoadState = "Failed to load the Zarf State from the cluster." - ErrSaveState = "Failed to save the Zarf State to the cluster." - ErrLoadPackageSecret = "Failed to load %s's secret from the cluster" - ErrNoClusterConnection = "Failed to connect to the cluster." - ErrTunnelFailed = "Failed to create a tunnel to the cluster." - ErrUnmarshal = "failed to unmarshal file: %w" - ErrWritingFile = "failed to write file %s: %s" - ErrDownloading = "failed to download %s: %s" - ErrCreatingDir = "failed to create directory %s: %s" - ErrRemoveFile = "failed to remove file %s: %s" - ErrUnarchive = "failed to unarchive %s: %s" - ErrConfirmCancel = "confirm selection canceled: %s" - ErrFileExtract = "failed to extract filename %s from archive %s: %s" - ErrFileNameExtract = "failed to extract filename from URL %s: %s" + ErrLoadState = "Failed to load the Zarf State from the cluster." + ErrSaveState = "Failed to save the Zarf State to the cluster." + ErrLoadPackageSecret = "Failed to load %s's secret from the cluster" + ErrNoClusterConnection = "Failed to connect to the cluster." + ErrTunnelFailed = "Failed to create a tunnel to the cluster." + ErrUnmarshal = "failed to unmarshal file: %w" + ErrWritingFile = "failed to write file %s: %s" + ErrDownloading = "failed to download %s: %s" + ErrCreatingDir = "failed to create directory %s: %s" + ErrRemoveFile = "failed to remove file %s: %s" + ErrUnarchive = "failed to unarchive %s: %s" + ErrConfirmCancel = "confirm selection canceled: %s" + ErrFileExtract = "failed to extract filename %s from archive %s: %s" + ErrFileNameExtract = "failed to extract filename from URL %s: %s" + ErrUnableToGenerateRandomSecret = "unable to generate a random secret" +) + +// Lint messages +const ( + UnsetVarLintWarning = "There are templates that are not set and won't be evaluated during lint" ) // Zarf CLI commands. @@ -211,6 +217,12 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA "This is called internally by the supported Gitea package component." CmdInternalArtifactRegistryGiteaTokenErr = "Unable to create an artifact registry token for the Gitea service." + CmdInternalUpdateGiteaPVCShort = "Updates an existing Gitea persistent volume claim" + CmdInternalUpdateGiteaPVCLong = "Updates an existing Gitea persistent volume claim by assessing if claim is a custom user provided claim or default." + + "This is called internally by the supported Gitea package component." + CmdInternalUpdateGiteaPVCErr = "Unable to update the existing Gitea persistent volume claim." + CmdInternalFlagUpdateGiteaPVCRollback = "Roll back previous Gitea persistent volume claim updates." + CmdInternalIsValidHostnameShort = "Checks if the current machine's hostname is RFC1123 compliant" CmdInternalIsValidHostnameErr = "The hostname '%s' is not valid. Ensure the hostname meets RFC1123 requirements https://www.rfc-editor.org/rfc/rfc1123.html." @@ -231,7 +243,7 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA "Kubernetes clusters are accessed via credentials in your current kubecontext defined in '~/.kube/config'" CmdPackageMirrorShort = "Mirrors a Zarf package's internal resources to specified image registries and git repositories" - CmdPackageMirrorLong = "Unpacks resources and dependencies from a Zarf package archive and mirrors them into the specified \n" + + CmdPackageMirrorLong = "Unpacks resources and dependencies from a Zarf package archive and mirrors them into the specified\n" + "image registries and git repositories within the target environment" CmdPackageMirrorExample = ` # Mirror resources to internal Zarf resources @@ -280,7 +292,7 @@ $ zarf package mirror-resources \ CmdPackageDeployFlagConfirm = "Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes." CmdPackageDeployFlagAdoptExistingResources = "Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover." CmdPackageDeployFlagSet = "Specify deployment variables to set on the command line (KEY=value)" - CmdPackageDeployFlagComponents = "Comma-separated list of components to install. Adding this flag will skip the init prompts for which components to install" + CmdPackageDeployFlagComponents = "Comma-separated list of components to deploy. Adding this flag will skip the prompts for selected components. Globbing component names with '*' and deselecting 'default' components with a leading '-' are also supported." CmdPackageDeployFlagShasum = "Shasum of the package to deploy. Required if deploying a remote package and \"--insecure\" is not provided" CmdPackageDeployFlagSget = "[Deprecated] Path to public sget key file for remote packages signed via cosign. This flag will be removed in v1.0.0 please use the --key flag instead." CmdPackageDeployFlagSkipWebhooks = "[alpha] Skip waiting for external webhooks to execute as each package component is deployed" @@ -290,7 +302,7 @@ $ zarf package mirror-resources \ CmdPackageDeployInvalidCLIVersionWarn = "CLIVersion is set to '%s' which can cause issues with package creation and deployment. To avoid such issues, please set the value to the valid semantic version for this version of Zarf." CmdPackageDeployErr = "Failed to deploy package: %s" - CmdPackageMirrorFlagComponents = "Comma-separated list of components to mirror. This list will be respected regardless of a component's 'required' status." + CmdPackageMirrorFlagComponents = "Comma-separated list of components to mirror. This list will be respected regardless of a component's 'required' or 'default' status. Globbing component names with '*' and deselecting components with a leading '-' are also supported." CmdPackageMirrorFlagNoChecksum = "Turns off the addition of a checksum to image tags (as would be used by the Zarf Agent) while mirroring images." CmdPackageInspectFlagSbom = "View SBOM contents while inspecting the package" @@ -299,7 +311,7 @@ $ zarf package mirror-resources \ CmdPackageRemoveShort = "Removes a Zarf package that has been deployed already (runs offline)" CmdPackageRemoveFlagConfirm = "REQUIRED. Confirm the removal action to prevent accidental deletions" - CmdPackageRemoveFlagComponents = "Comma-separated list of components to uninstall" + CmdPackageRemoveFlagComponents = "Comma-separated list of components to remove. This list will be respected regardless of a component's 'required' or 'default' status. Globbing component names with '*' and deselecting components with a leading '-' are also supported." CmdPackageRemoveTarballErr = "Invalid tarball path provided" CmdPackageRemoveExtractErr = "Unable to extract the package contents" CmdPackageRemoveErr = "Unable to remove the package with an error of: %s" @@ -318,8 +330,16 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPackagePublishFlagSigningKeyPassword = "Password to the private key file used for publishing packages" CmdPackagePublishErr = "Failed to publish package: %s" - CmdPackagePullShort = "Pulls a Zarf package from a remote registry and save to the local file system" - CmdPackagePullExample = "$ zarf package pull oci://my-registry.com/my-namespace/my-package:0.0.1-arm64" + CmdPackagePullShort = "Pulls a Zarf package from a remote registry and save to the local file system" + CmdPackagePullExample = ` +# Pull a package matching the current architecture +$ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 + +# Pull a package matching a specific architecture +$ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a arm64 + +# Pull a skeleton package +$ zarf package pull oci://ghcr.io/defenseunicorns/packages/dos-games:1.0.0 -a skeleton` CmdPackagePullFlagOutputDirectory = "Specify the output directory for the pulled Zarf package" CmdPackagePullErr = "Failed to pull package: %s" @@ -328,41 +348,46 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdPackageClusterSourceFallback = "%q does not satisfy any current sources, assuming it is a package deployed to a cluster" CmdPackageInvalidSource = "Unable to identify source from %q: %s" - // zarf prepare - CmdPrepareShort = "Tools to help prepare assets for packaging" + // zarf dev (prepare is an alias for dev) + CmdDevShort = "Commands useful for developing packages" + + CmdDevDeployShort = "[beta] Creates and deploys a Zarf package from a given directory" + CmdDevDeployLong = "[beta] Creates and deploys a Zarf package from a given directory, setting options like YOLO mode for faster iteration." + CmdDevDeployFlagNoYolo = "Disable the YOLO mode default override and create / deploy the package as-defined" + CmdDevDeployErr = "Failed to dev deploy: %s" - CmdPreparePatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + + CmdDevPatchGitShort = "Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:\n" + "This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook." - CmdPreparePatchGitOverwritePrompt = "Overwrite the file %s with these changes?" - CmdPreparePatchGitOverwriteErr = "Confirm overwrite canceled: %s" - CmdPreparePatchGitFileReadErr = "Unable to read the file %s" - CmdPreparePatchGitFileWriteErr = "Unable to write the changes back to the file" + CmdDevPatchGitOverwritePrompt = "Overwrite the file %s with these changes?" + CmdDevPatchGitOverwriteErr = "Confirm overwrite canceled: %s" + CmdDevPatchGitFileReadErr = "Unable to read the file %s" + CmdDevPatchGitFileWriteErr = "Unable to write the changes back to the file" - CmdPrepareSha256sumShort = "Generates a SHA256SUM for the given file" - CmdPrepareSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." - CmdPrepareSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" + CmdDevSha256sumShort = "Generates a SHA256SUM for the given file" + CmdDevSha256sumRemoteWarning = "This is a remote source. If a published checksum is available you should use that rather than calculating it directly from the remote link." + CmdDevSha256sumHashErr = "Unable to compute the SHA256SUM hash: %s" - CmdPrepareFindImagesShort = "Evaluates components in a zarf file to identify images specified in their helm charts and manifests" - CmdPrepareFindImagesLong = "Evaluates components in a zarf file to identify images specified in their helm charts and manifests.\n\n" + + CmdDevFindImagesShort = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests" + CmdDevFindImagesLong = "Evaluates components in a Zarf file to identify images specified in their helm charts and manifests.\n\n" + "Components that have repos that host helm charts can be processed by providing the --repo-chart-path." - CmdPrepareFindImagesErr = "Unable to find images: %s" + CmdDevFindImagesErr = "Unable to find images: %s" - CmdPrepareGenerateConfigShort = "Generates a config file for Zarf" - CmdPrepareGenerateConfigLong = "Generates a Zarf config file for controlling how the Zarf CLI operates. Optionally accepts a filename to write the config to.\n\n" + + CmdDevGenerateConfigShort = "Generates a config file for Zarf" + CmdDevGenerateConfigLong = "Generates a Zarf config file for controlling how the Zarf CLI operates. Optionally accepts a filename to write the config to.\n\n" + "The extension will determine the format of the config file, e.g. env-1.yaml, env-2.json, env-3.toml etc.\n" + "Accepted extensions are json, toml, yaml.\n\n" + "NOTE: This file must not already exist. If no filename is provided, the config will be written to the current working directory as zarf-config.toml." - CmdPrepareGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist" + CmdDevGenerateConfigErr = "Unable to write the config file %s, make sure the file doesn't already exist" - CmdPrepareFlagExtractPath = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")` - CmdPrepareFlagSet = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]." - CmdPrepareFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"` - CmdPrepareFlagGitAccount = "User or organization name for the git account that the repos are created under." - CmdPrepareFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" + CmdDevFlagExtractPath = `The path inside of an archive to use to calculate the sha256sum (i.e. for use with "files.extractPath")` + CmdDevFlagSet = "Specify package variables to set on the command line (KEY=value). Note, if using a config file, this will be set by [package.create.set]." + CmdDevFlagRepoChartPath = `If git repos hold helm charts, often found with gitops tools, specify the chart path, e.g. "/" or "/chart"` + CmdDevFlagGitAccount = "User or organization name for the git account that the repos are created under." + CmdDevFlagKubeVersion = "Override the default helm template KubeVersion when performing a package chart template" - CmdPrepareLintShort = "Verifies the package schema" - CmdPrepareLintLong = "Verifies the package schema and warns the user if they have variables that won't be evaluated" - CmdPrepareLintErr = "Unable to lint package: %s" + CmdDevLintShort = "Lints the given package for valid schema and recommended practices" + CmdDevLintLong = "Verifies the package schema, checks if any variables won't be evaluated, and checks for unpinned images/repos/files" + CmdDevLintErr = "Unable to lint package: %s" // zarf tools CmdToolsShort = "Collection of additional tools to make airgap easier" @@ -373,7 +398,7 @@ $ zarf package publish ./path/to/dir oci://my-registry.com/my-namespace CmdToolsArchiverDecompressShort = "Decompresses an archive or Zarf package based off of the source file extension." CmdToolsArchiverDecompressErr = "Unable to perform decompression: %s" - CmdToolsArchiverUnarchiveAllErr = "Unable to unarchive all nested tarballs" + CmdToolsArchiverUnarchiveAllErr = "Unable to unarchive all nested tarballs: %s" CmdToolsRegistryShort = "Tools for working with container registries using go-containertools" CmdToolsRegistryZarfState = "Retrieving registry information from Zarf state" @@ -558,6 +583,7 @@ $ zarf tools update-creds artifact --artifact-push-username={USERNAME} --artifac CmdToolsUpdateCredsUnableUpdateRegistry = "Unable to update Zarf Registry values: %s" CmdToolsUpdateCredsUnableUpdateGit = "Unable to update Zarf Git Server values: %s" CmdToolsUpdateCredsUnableUpdateAgent = "Unable to update Zarf Agent TLS secrets: %s" + CmdToolsUpdateCredsUnableUpdateCreds = "Unable to update Zarf credentials" // zarf version CmdVersionShort = "Shows the version of the running Zarf binary" @@ -597,9 +623,17 @@ const ( PkgCreateErrDifferentialSameVersion = "unable to create a differential package with the same version as the package you are using as a reference; the package version must be incremented" ) +// src/internal/packager/deploy. +const ( + PkgDeployErrMultipleComponentsSameGroup = "You cannot specify multiple components (%q, %q) within the same group (%q) when using the --components flag." + PkgDeployErrNoDefaultOrSelection = "You must make a selection from %q with the --components flag as there is no default in their group." + PkgDeployErrNoCompatibleComponentsForSelection = "No compatible components found that matched %q. Please check spelling and try again." + PkgDeployErrComponentSelectionCanceled = "Component selection canceled: %s" +) + // src/internal/packager/validate. const ( - PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." + PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" @@ -612,11 +646,14 @@ const ( PkgValidateErrChartNamespaceMissing = "chart %q must include a namespace" PkgValidateErrChartURLOrPath = "chart %q must have either a url or localPath" PkgValidateErrChartVersion = "chart %q must include a chart version" + PkgValidateErrComponentName = "component name %q must be all lowercase and contain no special characters except '-' and cannot start with a '-'" PkgValidateErrComponentNameNotUnique = "component name %q is not unique" PkgValidateErrComponent = "invalid component %q: %w" PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" + PkgValidateErrGroupMultipleDefaults = "group %q has multiple defaults (%q, %q)" + PkgValidateErrGroupOneComponent = "group %q only has one component (%q)" PkgValidateErrConstant = "invalid package constant: %w" PkgValidateErrImportDefinition = "invalid imported definition for %s: %s" PkgValidateErrInitNoYOLO = "sorry, you can't YOLO an init package" @@ -628,7 +665,7 @@ const ( PkgValidateErrName = "invalid package name: %w" PkgValidateErrPkgConstantName = "constant name %q must be all uppercase and contain no special characters except _" PkgValidateErrPkgConstantPattern = "provided value for constant %q does not match pattern %q" - PkgValidateErrPkgName = "package name %q must be all lowercase and contain no special characters except -" + PkgValidateErrPkgName = "package name %q must be all lowercase and contain no special characters except '-' and cannot start with a '-'" PkgValidateErrVariable = "invalid package variable: %w" PkgValidateErrYOLONoArch = "cluster architecture not allowed" PkgValidateErrYOLONoDistro = "cluster distros not allowed" diff --git a/src/internal/packager/git/clone.go b/src/internal/packager/git/clone.go index 464d3424f9..0123b81f2f 100644 --- a/src/internal/packager/git/clone.go +++ b/src/internal/packager/git/clone.go @@ -46,7 +46,7 @@ func (g *Git) clone(gitURL string, ref plumbing.ReferenceName, shallow bool) err // Clone the given repo. repo, err := git.PlainClone(g.GitPath, false, cloneOptions) if err != nil { - message.Warnf("Falling back to host 'git', failed to clone the repo with Zarf - %s: %s", gitURL, err.Error()) + message.Notef("Falling back to host 'git', failed to clone the repo %q with Zarf: %s", gitURL, err.Error()) return g.gitCloneFallback(gitURL, ref, shallow) } diff --git a/src/internal/packager/git/gitea.go b/src/internal/packager/git/gitea.go index 4aa1cf7b78..222ce3c5e6 100644 --- a/src/internal/packager/git/gitea.go +++ b/src/internal/packager/git/gitea.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "os" "time" netHttp "net/http" @@ -17,6 +18,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" + "k8s.io/apimachinery/pkg/runtime/schema" ) // CreateTokenResponse is the response given from creating a token in Gitea @@ -49,42 +52,6 @@ func (g *Git) CreateReadOnlyUser() error { tunnelURL := tunnel.HTTPEndpoint() - // Determine if the read only user already exists - getUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users", tunnelURL) - getUserRequest, _ := netHttp.NewRequest("GET", getUserEndpoint, nil) - out, err := g.DoHTTPThings(getUserRequest, g.Server.PushUsername, g.Server.PushPassword) - message.Debugf("GET %s:\n%s", getUserEndpoint, string(out)) - if err != nil { - return err - } - - hasReadOnlyUser := false - var users []map[string]interface{} - err = json.Unmarshal(out, &users) - if err != nil { - return err - } - - for _, user := range users { - if user["login"] == g.Server.PullUsername { - hasReadOnlyUser = true - } - } - - if hasReadOnlyUser { - // Update the existing user's password - updateUserBody := map[string]interface{}{ - "login_name": g.Server.PullUsername, - "password": g.Server.PullPassword, - } - updateUserData, _ := json.Marshal(updateUserBody) - updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PullUsername) - updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) - out, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) - message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) - return err - } - // Create json representation of the create-user request body createUserBody := map[string]interface{}{ "username": g.Server.PullUsername, @@ -97,12 +64,23 @@ func (g *Git) CreateReadOnlyUser() error { return err } + var out []byte + var statusCode int + // Send API request to create the user createUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users", tunnelURL) createUserRequest, _ := netHttp.NewRequest("POST", createUserEndpoint, bytes.NewBuffer(createUserData)) - out, err = g.DoHTTPThings(createUserRequest, g.Server.PushUsername, g.Server.PushPassword) + err = tunnel.Wrap(func() error { + out, statusCode, err = g.DoHTTPThings(createUserRequest, g.Server.PushUsername, g.Server.PushPassword) + return err + }) message.Debugf("POST %s:\n%s", createUserEndpoint, string(out)) if err != nil { + if statusCode == 422 { + message.Debugf("Read-only git user already exists. Skipping...") + return nil + } + return err } @@ -115,7 +93,65 @@ func (g *Git) CreateReadOnlyUser() error { updateUserData, _ := json.Marshal(updateUserBody) updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, g.Server.PullUsername) updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) - out, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) + err = tunnel.Wrap(func() error { + out, _, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, g.Server.PushPassword) + return err + }) + message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) + return err +} + +// UpdateZarfGiteaUsers updates Zarf gitea users +func (g *Git) UpdateZarfGiteaUsers(oldState *types.ZarfState) error { + + //Update git read only user password + err := g.UpdateGitUser(oldState.GitServer.PushPassword, g.Server.PullUsername, g.Server.PullPassword) + if err != nil { + return fmt.Errorf("unable to update gitea read only user password: %w", err) + } + + // Update Git admin password + err = g.UpdateGitUser(oldState.GitServer.PushPassword, g.Server.PushUsername, g.Server.PushPassword) + if err != nil { + return fmt.Errorf("unable to update gitea admin user password: %w", err) + } + return nil +} + +// UpdateGitUser updates Zarf git server users +func (g *Git) UpdateGitUser(oldAdminPass string, username string, userpass string) error { + message.Debugf("git.UpdateGitUser()") + + c, err := cluster.NewCluster() + if err != nil { + return err + } + // Establish a git tunnel to send the repo + tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, k8s.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort) + if err != nil { + return err + } + _, err = tunnel.Connect() + if err != nil { + return err + } + defer tunnel.Close() + tunnelURL := tunnel.HTTPEndpoint() + + var out []byte + + // Update the existing user's password + updateUserBody := map[string]interface{}{ + "login_name": username, + "password": userpass, + } + updateUserData, _ := json.Marshal(updateUserBody) + updateUserEndpoint := fmt.Sprintf("%s/api/v1/admin/users/%s", tunnelURL, username) + updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData)) + err = tunnel.Wrap(func() error { + out, _, err = g.DoHTTPThings(updateUserRequest, g.Server.PushUsername, oldAdminPass) + return err + }) message.Debugf("PATCH %s:\n%s", updateUserEndpoint, string(out)) return err } @@ -142,10 +178,15 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { tunnelURL := tunnel.Endpoint() + var out []byte + // Determine if the package token already exists getTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) getTokensRequest, _ := netHttp.NewRequest("GET", getTokensEndpoint, nil) - out, err := g.DoHTTPThings(getTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + err = tunnel.Wrap(func() error { + out, _, err = g.DoHTTPThings(getTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + return err + }) message.Debugf("GET %s:\n%s", getTokensEndpoint, string(out)) if err != nil { return CreateTokenResponse{}, err @@ -168,7 +209,10 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { // Delete the existing token to be replaced deleteTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens/%s", tunnelURL, g.Server.PushUsername, config.ZarfArtifactTokenName) deleteTokensRequest, _ := netHttp.NewRequest("DELETE", deleteTokensEndpoint, nil) - out, err := g.DoHTTPThings(deleteTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + err = tunnel.Wrap(func() error { + out, _, err = g.DoHTTPThings(deleteTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + return err + }) message.Debugf("DELETE %s:\n%s", deleteTokensEndpoint, string(out)) if err != nil { return CreateTokenResponse{}, err @@ -177,11 +221,15 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { createTokensEndpoint := fmt.Sprintf("http://%s/api/v1/users/%s/tokens", tunnelURL, g.Server.PushUsername) createTokensBody := map[string]interface{}{ - "name": config.ZarfArtifactTokenName, + "name": config.ZarfArtifactTokenName, + "scopes": []string{"read:user", "read:package", "write:package"}, } createTokensData, _ := json.Marshal(createTokensBody) createTokensRequest, _ := netHttp.NewRequest("POST", createTokensEndpoint, bytes.NewBuffer(createTokensData)) - out, err = g.DoHTTPThings(createTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + err = tunnel.Wrap(func() error { + out, _, err = g.DoHTTPThings(createTokensRequest, g.Server.PushUsername, g.Server.PushPassword) + return err + }) message.Debugf("POST %s:\n%s", createTokensEndpoint, string(out)) if err != nil { return CreateTokenResponse{}, err @@ -196,8 +244,36 @@ func (g *Git) CreatePackageRegistryToken() (CreateTokenResponse, error) { return createTokenResponse, nil } +// UpdateGiteaPVC updates the existing Gitea persistent volume claim and tells Gitea whether to create or not. +func UpdateGiteaPVC(shouldRollBack bool) (string, error) { + c, err := cluster.NewCluster() + if err != nil { + return "false", err + } + + pvcName := os.Getenv("ZARF_VAR_GIT_SERVER_EXISTING_PVC") + groupKind := schema.GroupKind{ + Group: "", + Kind: "PersistentVolumeClaim", + } + labels := map[string]string{"app.kubernetes.io/managed-by": "Helm"} + annotations := map[string]string{"meta.helm.sh/release-name": "zarf-gitea", "meta.helm.sh/release-namespace": "zarf"} + + if shouldRollBack { + err = c.K8s.RemoveLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) + return "false", err + } + + if pvcName == "data-zarf-gitea-0" { + err = c.K8s.AddLabelsAndAnnotations(cluster.ZarfNamespaceName, pvcName, groupKind, labels, annotations) + return "true", err + } + + return "false", err +} + // DoHTTPThings adds http request boilerplate and perform the request, checking for a successful response. -func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([]byte, error) { +func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([]byte, int, error) { message.Debugf("git.DoHttpThings()") // Prep the request with boilerplate @@ -209,17 +285,17 @@ func (g *Git) DoHTTPThings(request *netHttp.Request, username, secret string) ([ // Perform the request and get the response response, err := client.Do(request) if err != nil { - return []byte{}, err + return []byte{}, 0, err } responseBody, _ := io.ReadAll(response.Body) // If we get a 'bad' status code we will have no error, create a useful one to return if response.StatusCode < 200 || response.StatusCode >= 300 { err = fmt.Errorf("got status code of %d during http request with body of: %s", response.StatusCode, string(responseBody)) - return []byte{}, err + return []byte{}, response.StatusCode, err } - return responseBody, nil + return responseBody, response.StatusCode, nil } func (g *Git) addReadOnlyUserToRepo(tunnelURL, repo string) error { @@ -237,7 +313,7 @@ func (g *Git) addReadOnlyUserToRepo(tunnelURL, repo string) error { // Send API request to add a user as a read-only collaborator to a repo addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelURL, g.Server.PushUsername, repo, g.Server.PullUsername) addColabRequest, _ := netHttp.NewRequest("PUT", addColabEndpoint, bytes.NewBuffer(addColabData)) - out, err := g.DoHTTPThings(addColabRequest, g.Server.PushUsername, g.Server.PushPassword) + out, _, err := g.DoHTTPThings(addColabRequest, g.Server.PushUsername, g.Server.PushPassword) message.Debugf("PUT %s:\n%s", addColabEndpoint, string(out)) return err } diff --git a/src/internal/packager/git/push.go b/src/internal/packager/git/push.go index 0d6a6b942d..9bd52e2c79 100644 --- a/src/internal/packager/git/push.go +++ b/src/internal/packager/git/push.go @@ -50,8 +50,7 @@ func (g *Git) PushRepo(srcURL, targetFolder string) error { } if err := g.push(repo, spinner); err != nil { - spinner.Warnf("Unable to push the git repo %s (%s). Retrying....", repoFolder, err.Error()) - return err + return fmt.Errorf("failed to push the git repo %q: %w", repoFolder, err) } // Add the read-only user to this repo diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index ad5ff58315..7d89098ad3 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -55,11 +55,10 @@ func (h *Helm) newRenderer() (*renderer, error) { func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) { // This is very low cost and consistent for how we replace elsewhere, also good for debugging - tempDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + tempDir, err := utils.MakeTempDir(r.chartPath) if err != nil { return nil, fmt.Errorf("unable to create tmpdir: %w", err) } - defer os.RemoveAll(tempDir) path := filepath.Join(tempDir, "chart.yaml") // Write the context to a file for processing diff --git a/src/internal/packager/helm/zarf.go b/src/internal/packager/helm/zarf.go index 889468fdce..b8d42c3f84 100644 --- a/src/internal/packager/helm/zarf.go +++ b/src/internal/packager/helm/zarf.go @@ -7,7 +7,6 @@ package helm import ( "fmt" - "github.com/defenseunicorns/zarf/src/internal/packager/git" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -48,36 +47,6 @@ func (h *Helm) UpdateZarfRegistryValues() error { return nil } -// UpdateZarfGiteaValues updates the Zarf git server deployment with the new state values -func (h *Helm) UpdateZarfGiteaValues() error { - giteaValues := map[string]interface{}{ - "gitea": map[string]interface{}{ - "admin": map[string]interface{}{ - "username": h.cfg.State.GitServer.PushUsername, - "password": h.cfg.State.GitServer.PushPassword, - }, - }, - } - - h.chart = types.ZarfChart{ - Namespace: "zarf", - ReleaseName: "zarf-gitea", - } - - err := h.UpdateReleaseValues(giteaValues) - if err != nil { - return fmt.Errorf("error updating the release values: %w", err) - } - - g := git.New(h.cfg.State.GitServer) - err = g.CreateReadOnlyUser() - if err != nil { - return fmt.Errorf("unable to create the new Gitea read only user: %w", err) - } - - return nil -} - // UpdateZarfAgentValues updates the Zarf agent deployment with the new state values func (h *Helm) UpdateZarfAgentValues() error { spinner := message.NewProgressSpinner("Gathering information to update Zarf Agent TLS") diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 131d1128ac..ed115a1d65 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -30,7 +30,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/moby/moby/client" - "github.com/pterm/pterm" ) // ImgInfo wraps references/information about an image @@ -102,7 +101,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { img, hasImageLayers, err := i.PullImage(actualSrc, spinner) if err != nil { - metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to pull image %s: %w", actualSrc, err) + metadataImageConcurrency.ErrorChan <- fmt.Errorf("failed to pull %s: %w", actualSrc, err) return } @@ -121,7 +120,7 @@ func (i *ImageConfig) PullAll() ([]ImgInfo, error) { } onMetadataError := func(err error) error { - return fmt.Errorf("failed to load metadata for all images. This may be due to a network error or an invalid image reference: %w", err) + return err } if err := metadataImageConcurrency.WaitWithProgress(onMetadataProgress, onMetadataError); err != nil { @@ -446,13 +445,12 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im } } else if _, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure, i.Architectures...)...); err != nil { // If crane is unable to pull the image, try to load it from the local docker daemon. - message.Debugf("crane unable to pull image %s: %s", src, err) - spinner.Updatef("Falling back to docker for %s. This may take some time.", src) + message.Notef("Falling back to local 'docker' images, failed to find the manifest on a remote: %s", err.Error()) // Parse the image reference to get the image name. reference, err := name.ParseReference(src) if err != nil { - return nil, false, fmt.Errorf("failed to parse image reference %s: %w", src, err) + return nil, false, fmt.Errorf("failed to parse image reference: %w", err) } // Attempt to connect to the local docker daemon. @@ -466,33 +464,32 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im // Inspect the image to get the size. rawImg, _, err := cli.ImageInspectWithRaw(ctx, src) if err != nil { - return nil, false, fmt.Errorf("failed to inspect image %s via docker: %w", src, err) + return nil, false, fmt.Errorf("failed to inspect image via docker: %w", err) } // Warn the user if the image is large. if rawImg.Size > 750*1000*1000 { - warn := pterm.DefaultParagraph.WithMaxWidth(message.TermWidth).Sprintf("%s is %s and may take a very long time to load via docker. "+ + message.Warnf("%s is %s and may take a very long time to load via docker. "+ "See https://docs.zarf.dev/docs/faq for suggestions on how to improve large local image loading operations.", src, utils.ByteFormat(float64(rawImg.Size), 2)) - spinner.Warnf(warn) } // Use unbuffered opener to avoid OOM Kill issues https://github.com/defenseunicorns/zarf/issues/1214. // This will also take for ever to load large images. if img, err = daemon.Image(reference, daemon.WithUnbufferedOpener()); err != nil { - return nil, false, fmt.Errorf("failed to load image %s from docker daemon: %w", src, err) + return nil, false, fmt.Errorf("failed to load image from docker daemon: %w", err) } } else { // Manifest was found, so use crane to pull the image. if img, err = crane.Pull(src, config.GetCraneOptions(i.Insecure, i.Architectures...)...); err != nil { - return nil, false, fmt.Errorf("failed to pull image %s: %w", src, err) + return nil, false, fmt.Errorf("failed to pull image: %w", err) } cacheImage = true } hasImageLayers, err = utils.HasImageLayers(img) if err != nil { - return nil, false, fmt.Errorf("failed to check image %s layer mediatype: %w", src, err) + return nil, false, fmt.Errorf("failed to check image layer mediatype: %w", err) } if hasImageLayers && cacheImage { diff --git a/src/internal/packager/images/push.go b/src/internal/packager/images/push.go index 57482c0dd8..bc0e3aa4a8 100644 --- a/src/internal/packager/images/push.go +++ b/src/internal/packager/images/push.go @@ -78,6 +78,14 @@ func (i *ImageConfig) PushToZarfRegistry() error { defer tunnel.Close() } + pushImage := func(img v1.Image, name string) error { + if tunnel != nil { + return tunnel.Wrap(func() error { return crane.Push(img, name, pushOptions...) }) + } + + return crane.Push(img, name, pushOptions...) + } + for refInfo, img := range refInfoToImage { refTruncated := message.Truncate(refInfo.Reference, 55, true) progressBar.UpdateTitle(fmt.Sprintf("Pushing %s", refTruncated)) @@ -91,7 +99,8 @@ func (i *ImageConfig) PushToZarfRegistry() error { message.Debugf("crane.Push() %s:%s -> %s)", i.ImagesPath, refInfo.Reference, offlineNameCRC) - if err = crane.Push(img, offlineNameCRC, pushOptions...); err != nil { + err = pushImage(img, offlineNameCRC) + if err != nil { return err } } @@ -105,7 +114,8 @@ func (i *ImageConfig) PushToZarfRegistry() error { message.Debugf("crane.Push() %s:%s -> %s)", i.ImagesPath, refInfo.Reference, offlineName) - if err = crane.Push(img, offlineName, pushOptions...); err != nil { + err = pushImage(img, offlineName) + if err != nil { return err } } diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 9a1d57db1a..dea997b635 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -8,19 +8,17 @@ import ( "fmt" "path/filepath" "regexp" - "strings" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) var ( - // IsLowercaseNumberHyphen is a regex for lowercase, numbers and hyphens. - // https://regex101.com/r/FLdG9G/1 - IsLowercaseNumberHyphen = regexp.MustCompile(`^[a-z0-9\-]+$`).MatchString + // IsLowercaseNumberHyphenNoStartHyphen is a regex for lowercase, numbers and hyphens that cannot start with a hyphen. + // https://regex101.com/r/FLdG9G/2 + IsLowercaseNumberHyphenNoStartHyphen = regexp.MustCompile(`^[a-z0-9][a-z0-9\-]*$`).MatchString // IsUppercaseNumberUnderscore is a regex for uppercase, numbers and underscores. // https://regex101.com/r/tfsEuZ/1 IsUppercaseNumberUnderscore = regexp.MustCompile(`^[A-Z0-9_]+$`).MatchString @@ -49,6 +47,8 @@ func Run(pkg types.ZarfPackage) error { } uniqueComponentNames := make(map[string]bool) + groupDefault := make(map[string]string) + groupedComponents := make(map[string][]string) for _, component := range pkg.Components { // ensure component name is unique @@ -60,6 +60,23 @@ func Run(pkg types.ZarfPackage) error { if err := validateComponent(pkg, component); err != nil { return fmt.Errorf(lang.PkgValidateErrComponent, component.Name, err) } + + // ensure groups don't have multiple defaults or only one component + if component.Group != "" { + if component.Default { + if _, ok := groupDefault[component.Group]; ok { + return fmt.Errorf(lang.PkgValidateErrGroupMultipleDefaults, component.Group, groupDefault[component.Group], component.Name) + } + groupDefault[component.Group] = component.Name + } + groupedComponents[component.Group] = append(groupedComponents[component.Group], component.Name) + } + } + + for groupKey, componentNames := range groupedComponents { + if len(componentNames) == 1 { + return fmt.Errorf(lang.PkgValidateErrGroupOneComponent, groupKey, componentNames[0]) + } } return nil @@ -94,9 +111,6 @@ func ImportDefinition(component *types.ZarfComponent) error { if !ok { return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "URL is not a valid OCI URL") } - if !strings.HasSuffix(url, oci.SkeletonSuffix) { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "OCI import URL must end with -skeleton") - } } return nil @@ -111,6 +125,10 @@ func oneIfNotEmpty(testString string) int { } func validateComponent(pkg types.ZarfPackage, component types.ZarfComponent) error { + if !IsLowercaseNumberHyphenNoStartHyphen(component.Name) { + return fmt.Errorf(lang.PkgValidateErrComponentName, component.Name) + } + if component.Required { if component.Default { return fmt.Errorf(lang.PkgValidateErrComponentReqDefault, component.Name) @@ -254,7 +272,7 @@ func validateYOLO(component types.ZarfComponent) error { } func validatePackageName(subject string) error { - if !IsLowercaseNumberHyphen(subject) { + if !IsLowercaseNumberHyphenNoStartHyphen(subject) { return fmt.Errorf(lang.PkgValidateErrPkgName, subject) } diff --git a/src/pkg/cluster/injector.go b/src/pkg/cluster/injector.go index a472b18205..a740db617d 100644 --- a/src/pkg/cluster/injector.go +++ b/src/pkg/cluster/injector.go @@ -245,7 +245,15 @@ func (c *Cluster) injectorIsReady(seedImages []transform.Image, spinner *message for _, seedImage := range seedImages { seedRegistry := fmt.Sprintf("%s/v2/%s/manifests/%s", tunnel.HTTPEndpoint(), seedImage.Path, seedImage.Tag) - if resp, err := http.Get(seedRegistry); err != nil || resp.StatusCode != 200 { + + var resp *http.Response + var err error + err = tunnel.Wrap(func() error { + resp, err = http.Get(seedRegistry) + return err + }) + + if err != nil || resp.StatusCode != 200 { // Just debug log the output because failures just result in trying the next image message.Debug(resp, err) return false diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index b9025b7b47..5fe6e4d9dd 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -12,13 +12,13 @@ import ( "slices" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" "github.com/defenseunicorns/zarf/src/pkg/k8s" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/pki" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -72,7 +72,9 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { // Defaults state.Distro = distro - state.LoggingSecret = utils.RandomString(config.ZarfGeneratedPasswordLen) + if state.LoggingSecret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } // Setup zarf agent PKI state.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) @@ -110,8 +112,12 @@ func (c *Cluster) InitZarfState(initOptions types.ZarfInitOptions) error { return fmt.Errorf("unable get default Zarf service account: %w", err) } - state.GitServer = c.fillInEmptyGitServerValues(initOptions.GitServer) - state.RegistryInfo = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo) + if state.GitServer, err = c.fillInEmptyGitServerValues(initOptions.GitServer); err != nil { + return err + } + if state.RegistryInfo, err = c.fillInEmptyContainerRegistryValues(initOptions.RegistryInfo); err != nil { + return err + } state.ArtifactServer = c.fillInEmptyArtifactServerValues(initOptions.ArtifactServer) } else { if helpers.IsNotZeroAndNotEqual(initOptions.GitServer, state.GitServer) { @@ -158,7 +164,7 @@ func (c *Cluster) LoadZarfState() (state *types.ZarfState, err error) { // Set up the API connection secret, err := c.GetSecret(ZarfNamespaceName, ZarfStateSecretName) if err != nil { - return nil, fmt.Errorf("%w. %s", err, utils.ColorWrap("Did you remember to zarf init?", color.Bold)) + return nil, fmt.Errorf("%w. %s", err, message.ColorWrap("Did you remember to zarf init?", color.Bold)) } err = json.Unmarshal(secret.Data[ZarfStateDataKey], &state) @@ -245,9 +251,9 @@ func (c *Cluster) SaveZarfState(state *types.ZarfState) error { } // MergeZarfState merges init options for provided services into the provided state to create a new state struct -func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) *types.ZarfState { +func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions, services []string) (*types.ZarfState, error) { newState := *oldState - + var err error if slices.Contains(services, message.RegistryKey) { newState.RegistryInfo = helpers.MergeNonZero(newState.RegistryInfo, initOptions.RegistryInfo) // Set the state of the internal registry if it has changed @@ -259,10 +265,14 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.RegistryInfo.PushPassword == oldState.RegistryInfo.PushPassword && oldState.RegistryInfo.InternalRegistry { - newState.RegistryInfo.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.RegistryInfo.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } if newState.RegistryInfo.PullPassword == oldState.RegistryInfo.PullPassword && oldState.RegistryInfo.InternalRegistry { - newState.RegistryInfo.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.RegistryInfo.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } } if slices.Contains(services, message.GitKey) { @@ -277,10 +287,14 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za // Set the new passwords if they should be autogenerated if newState.GitServer.PushPassword == oldState.GitServer.PushPassword && oldState.GitServer.InternalServer { - newState.GitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.GitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } if newState.GitServer.PullPassword == oldState.GitServer.PullPassword && oldState.GitServer.InternalServer { - newState.GitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if newState.GitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } } if slices.Contains(services, message.ArtifactKey) { @@ -302,10 +316,11 @@ func (c *Cluster) MergeZarfState(oldState *types.ZarfState, initOptions types.Za newState.AgentTLS = pki.GeneratePKI(config.ZarfAgentHost) } - return &newState + return &newState, nil } -func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) types.RegistryInfo { +func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.RegistryInfo) (types.RegistryInfo, error) { + var err error // Set default NodePort if none was provided if containerRegistry.NodePort == 0 { containerRegistry.NodePort = config.ZarfInClusterContainerRegistryNodePort @@ -319,7 +334,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg // Generate a push-user password if not provided by init flag if containerRegistry.PushPassword == "" { - containerRegistry.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if containerRegistry.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } // Set pull-username if not provided by init flag @@ -333,7 +350,9 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg } if containerRegistry.PullPassword == "" { if containerRegistry.InternalRegistry { - containerRegistry.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if containerRegistry.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } else { // If this is an external registry and a pull-user wasn't provided, use the same credentials as the push user containerRegistry.PullPassword = containerRegistry.PushPassword @@ -341,14 +360,17 @@ func (c *Cluster) fillInEmptyContainerRegistryValues(containerRegistry types.Reg } if containerRegistry.Secret == "" { - containerRegistry.Secret = utils.RandomString(config.ZarfGeneratedSecretLen) + if containerRegistry.Secret, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return containerRegistry, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } - return containerRegistry + return containerRegistry, nil } // Fill in empty GitServerInfo values with the defaults. -func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) types.GitServerInfo { +func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) (types.GitServerInfo, error) { + var err error // Set default svc url if an external repository was not provided if gitServer.Address == "" { gitServer.Address = config.ZarfInClusterGitServiceURL @@ -357,7 +379,9 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type // Generate a push-user password if not provided by init flag if gitServer.PushPassword == "" { - gitServer.PushPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if gitServer.PushPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return gitServer, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } // Set read-user information if using an internal repository, otherwise copy from the push-user @@ -370,13 +394,15 @@ func (c *Cluster) fillInEmptyGitServerValues(gitServer types.GitServerInfo) type } if gitServer.PullPassword == "" { if gitServer.InternalServer { - gitServer.PullPassword = utils.RandomString(config.ZarfGeneratedPasswordLen) + if gitServer.PullPassword, err = helpers.RandomString(config.ZarfGeneratedPasswordLen); err != nil { + return gitServer, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err) + } } else { gitServer.PullPassword = gitServer.PushPassword } } - return gitServer + return gitServer, nil } // Fill in empty ArtifactServerInfo values with the defaults. diff --git a/src/pkg/interactive/components.go b/src/pkg/interactive/components.go new file mode 100644 index 0000000000..bb8f244f75 --- /dev/null +++ b/src/pkg/interactive/components.go @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package interactive contains functions for interacting with the user via STDIN. +package interactive + +import ( + "fmt" + "strings" + + "github.com/AlecAivazis/survey/v2" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + "github.com/pterm/pterm" +) + +// SelectOptionalComponent prompts to confirm optional components +func SelectOptionalComponent(component types.ZarfComponent) (confirmComponent bool) { + // Confirm flag passed, just use defaults + if config.CommonOptions.Confirm { + return component.Default + } + + message.HorizontalRule() + + displayComponent := component + displayComponent.Description = "" + utils.ColorPrintYAML(displayComponent, nil, false) + if component.Description != "" { + message.Question(component.Description) + } + + prompt := &survey.Confirm{ + Message: fmt.Sprintf("Deploy the %s component?", component.Name), + Default: component.Default, + } + if err := survey.AskOne(prompt, &confirmComponent); err != nil { + message.Fatalf(nil, lang.PkgDeployErrComponentSelectionCanceled, err.Error()) + } + + return confirmComponent +} + +// SelectChoiceGroup prompts to select component groups +func SelectChoiceGroup(componentGroup []types.ZarfComponent) types.ZarfComponent { + // Confirm flag passed, just use defaults + if config.CommonOptions.Confirm { + var componentNames []string + for _, component := range componentGroup { + // If the component is default, then return it + if component.Default { + return component + } + // Add each component name to the list + componentNames = append(componentNames, component.Name) + } + // If no default component was found, give up + message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) + } + + message.HorizontalRule() + + var chosen int + var options []string + + for _, component := range componentGroup { + text := fmt.Sprintf("Name: %s\n Description: %s\n", component.Name, component.Description) + options = append(options, text) + } + + prompt := &survey.Select{ + Message: "Select a component to deploy:", + Options: options, + } + + pterm.Println() + + if err := survey.AskOne(prompt, &chosen); err != nil { + message.Fatalf(nil, lang.PkgDeployErrComponentSelectionCanceled, err.Error()) + } + + return componentGroup[chosen] +} diff --git a/src/pkg/k8s/dynamic.go b/src/pkg/k8s/dynamic.go index 51a1a29e90..daf87c7a1a 100644 --- a/src/pkg/k8s/dynamic.go +++ b/src/pkg/k8s/dynamic.go @@ -16,6 +16,16 @@ import ( // AddLabelsAndAnnotations adds the provided labels and annotations to the specified K8s resource func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string) error { + return k.updateLabelsAndAnnotations(resourceNamespace, resourceName, groupKind, labels, annotations, false) +} + +// RemoveLabelsAndAnnotations removes the provided labels and annotations to the specified K8s resource +func (k *K8s) RemoveLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string) error { + return k.updateLabelsAndAnnotations(resourceNamespace, resourceName, groupKind, labels, annotations, true) +} + +// updateLabelsAndAnnotations updates the provided labels and annotations to the specified K8s resource +func (k *K8s) updateLabelsAndAnnotations(resourceNamespace string, resourceName string, groupKind schema.GroupKind, labels map[string]string, annotations map[string]string, isRemove bool) error { dynamicClient := dynamic.NewForConfigOrDie(k.RestConfig) discoveryClient := discovery.NewDiscoveryClientForConfigOrDie(k.RestConfig) @@ -43,7 +53,11 @@ func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName str deployedLabels = make(map[string]string) } for key, value := range labels { - deployedLabels[key] = value + if isRemove { + delete(deployedLabels, key) + } else { + deployedLabels[key] = value + } } deployedResource.SetLabels(deployedLabels) @@ -55,7 +69,11 @@ func (k *K8s) AddLabelsAndAnnotations(resourceNamespace string, resourceName str deployedAnnotations = make(map[string]string) } for key, value := range annotations { - deployedAnnotations[key] = value + if isRemove { + delete(deployedAnnotations, key) + } else { + deployedAnnotations[key] = value + } } deployedResource.SetAnnotations(deployedAnnotations) diff --git a/src/pkg/k8s/pods.go b/src/pkg/k8s/pods.go index 486ff075af..e4853034e0 100644 --- a/src/pkg/k8s/pods.go +++ b/src/pkg/k8s/pods.go @@ -109,63 +109,59 @@ func (k *K8s) WaitForPodsAndContainers(target PodLookup, include PodFilter) []co var readyPods = []corev1.Pod{} - // Reverse sort by creation time + // Sort the pods from newest to oldest sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].CreationTimestamp.After(pods.Items[j].CreationTimestamp.Time) }) - if len(pods.Items) > 0 { - for _, pod := range pods.Items { - k.Log("Testing pod %q", pod.Name) + for _, pod := range pods.Items { + k.Log("Testing pod %q", pod.Name) - // If an include function is provided, only keep pods that return true - if include != nil && !include(pod) { - continue - } - - // Handle container targeting - if target.Container != "" { - k.Log("Testing pod %q for container %q", pod.Name, target.Container) - var matchesInitContainer bool - - // Check the status of initContainers for a running match - for _, initContainer := range pod.Status.InitContainerStatuses { - isRunning := initContainer.State.Running != nil - if isRunning && initContainer.Name == target.Container { - // On running match in initContainer break this loop - matchesInitContainer = true - readyPods = append(readyPods, pod) - break - } - } + // If an include function is provided, only keep pods that return true + if include != nil && !include(pod) { + continue + } - // Don't check any further if there's already a match - if matchesInitContainer { - continue + // Handle container targeting + if target.Container != "" { + k.Log("Testing pod %q for container %q", pod.Name, target.Container) + var matchesInitContainer bool + + // Check the status of initContainers for a running match + for _, initContainer := range pod.Status.InitContainerStatuses { + isRunning := initContainer.State.Running != nil + if isRunning && initContainer.Name == target.Container { + // On running match in initContainer break this loop + matchesInitContainer = true + readyPods = append(readyPods, pod) + break } + } - // Check the status of regular containers for a running match - for _, container := range pod.Status.ContainerStatuses { - isRunning := container.State.Running != nil - if isRunning && container.Name == target.Container { - readyPods = append(readyPods, pod) - } - } + // Don't check any further if there's already a match + if matchesInitContainer { + continue + } - } else { - status := pod.Status.Phase - k.Log("Testing pod %q phase, want (%q) got (%q)", pod.Name, corev1.PodRunning, status) - // Regular status checking without a container - if status == corev1.PodRunning { + // Check the status of regular containers for a running match + for _, container := range pod.Status.ContainerStatuses { + isRunning := container.State.Running != nil + if isRunning && container.Name == target.Container { readyPods = append(readyPods, pod) } } - + } else { + status := pod.Status.Phase + k.Log("Testing pod %q phase, want (%q) got (%q)", pod.Name, corev1.PodRunning, status) + // Regular status checking without a container + if status == corev1.PodRunning { + readyPods = append(readyPods, pod) + } } + } - if len(readyPods) > 0 { - return readyPods - } + if len(readyPods) > 0 { + return readyPods } time.Sleep(3 * time.Second) diff --git a/src/pkg/k8s/tunnel.go b/src/pkg/k8s/tunnel.go index 6ca835d351..116db22ab7 100644 --- a/src/pkg/k8s/tunnel.go +++ b/src/pkg/k8s/tunnel.go @@ -40,6 +40,7 @@ type Tunnel struct { attempt int stopChan chan struct{} readyChan chan struct{} + errChan chan error } // NewTunnel will create a new Tunnel struct. @@ -60,6 +61,23 @@ func (k *K8s) NewTunnel(namespace, resourceType, resourceName, urlSuffix string, }, nil } +// Wrap takes a function that returns an error and wraps it to check for tunnel errors as well. +func (tunnel *Tunnel) Wrap(function func() error) error { + var err error + funcErrChan := make(chan error) + + go func() { + funcErrChan <- function() + }() + + select { + case err = <-funcErrChan: + return err + case err = <-tunnel.ErrChan(): + return err + } +} + // Connect will establish a tunnel to the specified target. func (tunnel *Tunnel) Connect() (string, error) { url, err := tunnel.establish() @@ -90,6 +108,11 @@ func (tunnel *Tunnel) Endpoint() string { return fmt.Sprintf("%s:%d", helpers.IPV4Localhost, tunnel.localPort) } +// ErrChan returns the tunnel's error channel +func (tunnel *Tunnel) ErrChan() chan error { + return tunnel.errChan +} + // HTTPEndpoint returns the tunnel endpoint as a HTTP URL string. func (tunnel *Tunnel) HTTPEndpoint() string { return fmt.Sprintf("http://%s", tunnel.Endpoint()) @@ -189,6 +212,9 @@ func (tunnel *Tunnel) establish() (string, error) { tunnel.localPort = localPort url := tunnel.FullURL() + // Store the error channel to listen for errors + tunnel.errChan = errChan + tunnel.kube.Log("Creating port forwarding tunnel at %s", url) return url, nil } diff --git a/src/pkg/layout/component.go b/src/pkg/layout/component.go index 2309d8200c..43123e65dc 100644 --- a/src/pkg/layout/component.go +++ b/src/pkg/layout/component.go @@ -65,7 +65,7 @@ func (c *Components) Archive(component types.ZarfComponent, cleanupTemp bool) (e if size > 0 { tb := fmt.Sprintf("%s.tar", base) message.Debugf("Archiving %q", name) - if err := archiver.Archive([]string{base}, tb); err != nil { + if err := utils.CreateReproducibleTarballFromDir(base, name, tb); err != nil { return err } if c.Tarballs == nil { diff --git a/src/pkg/layout/sbom.go b/src/pkg/layout/sbom.go index 8920331a52..13f7ee0fc1 100644 --- a/src/pkg/layout/sbom.go +++ b/src/pkg/layout/sbom.go @@ -60,14 +60,9 @@ func (s *SBOMs) Archive() (err error) { dir := s.Path tb := filepath.Join(filepath.Dir(dir), SBOMTar) - allSBOMFiles, err := filepath.Glob(filepath.Join(dir, "*")) - if err != nil { + if err := utils.CreateReproducibleTarballFromDir(dir, "", tb); err != nil { return err } - - if err = archiver.Archive(allSBOMFiles, tb); err != nil { - return - } s.Path = tb return os.RemoveAll(dir) } diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index ef1d2e235f..b4b6834a0f 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/defenseunicorns/zarf/src/config" + "github.com/fatih/color" "github.com/pterm/pterm" "github.com/sergi/go-diff/diffmatchpatch" ) @@ -322,15 +324,20 @@ func Truncate(text string, length int, invert bool) string { func Table(header []string, data [][]string) { pterm.Println() - if len(header) > 0 { - header[0] = fmt.Sprintf(" %s", header[0]) + // To avoid side effects make copies of the header and data before adding padding + headerCopy := make([]string, len(header)) + copy(headerCopy, header) + dataCopy := make([][]string, len(data)) + copy(dataCopy, data) + if len(headerCopy) > 0 { + headerCopy[0] = fmt.Sprintf(" %s", headerCopy[0]) } table := pterm.TableData{ - header, + headerCopy, } - for _, row := range data { + for _, row := range dataCopy { if len(row) > 0 { row[0] = fmt.Sprintf(" %s", row[0]) } @@ -340,6 +347,25 @@ func Table(header []string, data [][]string) { pterm.DefaultTable.WithHasHeader().WithData(table).Render() } +// ColorWrap changes a string to an ansi color code and appends the default color to the end +// preventing future characters from taking on the given color +// returns string as normal if color is disabled +func ColorWrap(str string, attr color.Attribute) string { + if config.NoColor { + return str + } + return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) +} + +// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. +func First30last30(s string) string { + if len(s) > 60 { + return s[0:27] + "..." + s[len(s)-26:] + } + + return s +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) diff --git a/src/pkg/message/spinner.go b/src/pkg/message/spinner.go index 8391e6633d..cb2ea49a3a 100644 --- a/src/pkg/message/spinner.go +++ b/src/pkg/message/spinner.go @@ -129,19 +129,9 @@ func (p *Spinner) Successf(format string, a ...any) { p.Stop() } -// Warnf prints a warning message with the spinner. -func (p *Spinner) Warnf(format string, a ...any) { - text := pterm.Sprintf(format, a...) - if p.spinner != nil { - p.spinner.Warning(text) - } else { - Warn(text) - } -} - // Errorf prints an error message with the spinner. func (p *Spinner) Errorf(err error, format string, a ...any) { - p.Warnf(format, a...) + Warnf(format, a...) debugPrinter(2, err) } diff --git a/src/pkg/oci/common.go b/src/pkg/oci/common.go index 641f53e989..c228ec0675 100644 --- a/src/pkg/oci/common.go +++ b/src/pkg/oci/common.go @@ -16,6 +16,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config/configfile" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/registry" "oras.land/oras-go/v2/registry/remote" @@ -27,23 +28,80 @@ const ( ZarfLayerMediaTypeBlob = "application/vnd.zarf.layer.v1.blob" // ZarfConfigMediaType is the media type for the manifest config ZarfConfigMediaType = "application/vnd.zarf.config.v1+json" - // SkeletonSuffix is the reference suffix used for skeleton packages - SkeletonSuffix = "skeleton" + // SkeletonArch is the architecture used for skeleton packages + SkeletonArch = "skeleton" + // MultiOS is the OS used for multi-platform packages + MultiOS = "multi" ) // OrasRemote is a wrapper around the Oras remote repository that includes a progress bar for interactive feedback. type OrasRemote struct { - repo *remote.Repository - root *ZarfOCIManifest - ctx context.Context - Transport *utils.Transport - CopyOpts oras.CopyOptions + repo *remote.Repository + root *ZarfOCIManifest + ctx context.Context + Transport *utils.Transport + CopyOpts oras.CopyOptions + targetPlatform *ocispec.Platform +} + +// Modifier is a function that modifies an OrasRemote +type Modifier func(*OrasRemote) + +// WithContext sets the context for the remote +func WithContext(ctx context.Context) Modifier { + return func(o *OrasRemote) { + o.ctx = ctx + } +} + +// WithCopyOpts sets the copy options for the remote +func WithCopyOpts(opts oras.CopyOptions) Modifier { + return func(o *OrasRemote) { + o.CopyOpts = opts + } +} + +// WithPlainHTTP sets the plain HTTP flag for the remote +func WithPlainHTTP(plainHTTP bool) Modifier { + return func(o *OrasRemote) { + o.repo.PlainHTTP = plainHTTP + } +} + +// WithInsecureSkipVerify sets the insecure TLS flag for the remote +func WithInsecureSkipVerify(insecure bool) Modifier { + return func(o *OrasRemote) { + o.Transport.Base.(*http.Transport).TLSClientConfig.InsecureSkipVerify = insecure + } +} + +// WithTargetPlatform sets the target platform for the remote +func WithTargetPlatform(platform *ocispec.Platform) Modifier { + return func(o *OrasRemote) { + o.targetPlatform = platform + } +} + +// WithSkeletonArch sets the target architecture for the remote to skeleton +func WithSkeletonArch() Modifier { + return WithTargetPlatform(&ocispec.Platform{ + OS: MultiOS, + Architecture: SkeletonArch, + }) +} + +// WithArch sets the target architecture for the remote +func WithArch(arch string) Modifier { + return WithTargetPlatform(&ocispec.Platform{ + OS: MultiOS, + Architecture: arch, + }) } // NewOrasRemote returns an oras remote repository client and context for the given url. // // Registry auth is handled by the Docker CLI's credential store and checked before returning the client -func NewOrasRemote(url string) (*OrasRemote, error) { +func NewOrasRemote(url string, mods ...Modifier) (*OrasRemote, error) { ref, err := registry.ParseReference(strings.TrimPrefix(url, helpers.OCIURLPrefix)) if err != nil { return nil, fmt.Errorf("failed to parse OCI reference %q: %w", url, err) @@ -59,11 +117,32 @@ func NewOrasRemote(url string) (*OrasRemote, error) { copyOpts.PostCopy = o.printLayerCopied o.CopyOpts = copyOpts - o.WithContext(context.TODO()) - o.WithInsecureConnection(zarfconfig.CommonOptions.Insecure) + // right now --insecure is overloaded to mean both plain HTTP and insecure TLS + // putting this here as the "default" for the remote + // but can be overridden by a provided modifier + insecureMod := WithInsecureSkipVerify(zarfconfig.CommonOptions.Insecure) + insecureMod(o) + + httpMod := WithPlainHTTP(zarfconfig.CommonOptions.Insecure) + httpMod(o) + + for _, mod := range mods { + mod(o) + } + + // if no context is provided, use the default + if o.ctx == nil { + o.ctx = context.TODO() + } + return o, nil } +// Repo gives you access to the underlying remote repository +func (o *OrasRemote) Repo() *remote.Repository { + return o.repo +} + // setRepository sets the repository for the remote as well as the auth client. func (o *OrasRemote) setRepository(ref registry.Reference) error { o.root = nil @@ -92,11 +171,6 @@ func (o *OrasRemote) setRepository(ref registry.Reference) error { return nil } -// WithContext sets the context for the remote -func (o *OrasRemote) WithContext(ctx context.Context) { - o.ctx = ctx -} - // createAuthClient returns an auth client for the given reference. // // The credentials are pulled using Docker's default credential store. @@ -148,14 +222,3 @@ func (o *OrasRemote) createAuthClient(ref registry.Reference) (*auth.Client, err return client, nil } - -// WithInsecureConnection sets the insecure connection flag for the remote -func (o *OrasRemote) WithInsecureConnection(insecure bool) { - o.repo.PlainHTTP = insecure - o.Transport.Base.(*http.Transport).TLSClientConfig.InsecureSkipVerify = insecure -} - -// Repo gives you access to the underlying remote repository -func (o *OrasRemote) Repo() *remote.Repository { - return o.repo -} diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index 4306e709e1..1b95e61efd 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -11,6 +11,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" goyaml "github.com/goccy/go-yaml" @@ -18,7 +19,26 @@ import ( // ResolveRoot returns the root descriptor for the remote repository func (o *OrasRemote) ResolveRoot() (ocispec.Descriptor, error) { - return o.repo.Resolve(o.ctx, o.repo.Reference.Reference) + // first try to resolve the reference into an OCI descriptor directly + desc, err := o.repo.Resolve(o.ctx, o.repo.Reference.Reference) + // if we succeeded and it's not an index, return it + // otherwise we will use oras.Resolve which will fetch the index, then resolve the manifest + // w/ the target platform + // + // this error is purposefully ignored, as we want to try oras.Resolve if the first attempt fails + if err == nil && desc.MediaType != ocispec.MediaTypeImageIndex { + return desc, nil + } + + if o.targetPlatform == nil && desc.MediaType == ocispec.MediaTypeImageIndex { + return ocispec.Descriptor{}, fmt.Errorf("%q resolved to an image index, but no target platform was specified", o.repo.Reference.Reference) + } + + resolveOpts := oras.ResolveOptions{ + TargetPlatform: o.targetPlatform, + } + // if the first attempt failed to resolve, or returned an index, try again with oras.Resolve + return oras.Resolve(o.ctx, o.repo, o.repo.Reference.Reference, resolveOpts) } // FetchRoot fetches the root manifest from the remote repository. diff --git a/src/pkg/oci/progress.go b/src/pkg/oci/progress.go new file mode 100644 index 0000000000..2c0ad4769b --- /dev/null +++ b/src/pkg/oci/progress.go @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package oci contains functions for interacting with Zarf packages stored in OCI registries. +package oci + +import ( + "context" + "fmt" + + "github.com/defenseunicorns/zarf/src/pkg/message" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// printLayerSkipped prints a debug message when a layer has been successfully skipped. +func (o *OrasRemote) printLayerSkipped(_ context.Context, desc ocispec.Descriptor) error { + return o.printLayer(desc, "skipped") +} + +// printLayerCopied prints a debug message when a layer has been successfully copied to/from a registry. +func (o *OrasRemote) printLayerCopied(_ context.Context, desc ocispec.Descriptor) error { + return o.printLayer(desc, "copied") +} + +// printLayer prints a debug message when a layer has been successfully published/pulled to/from a registry. +func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { + title := desc.Annotations[ocispec.AnnotationTitle] + var layerInfo string + if title != "" { + layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], message.First30last30(title)) + } else { + layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) + } + message.Debugf("%s (%s)", layerInfo, suffix) + return nil +} diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 310778c78c..8bebcee533 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -217,6 +217,20 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta if err != nil { return nil, err } + if desc.MediaType == ocispec.MediaTypeImageIndex { + manifestDescs := nodes + nodes = []ocispec.Descriptor{} + // expand the manifests + for _, node := range manifestDescs { + manifest, err := o.FetchManifest(node) + if err != nil { + return nil, err + } + nodes = append(nodes, manifest.Layers...) + nodes = append(nodes, manifest.Config) + } + } + var ret []ocispec.Descriptor for _, node := range nodes { if slices.Contains(shas, node.Digest.Encoded()) { diff --git a/src/pkg/oci/push.go b/src/pkg/oci/push.go index 2d057003f1..958419c8a2 100644 --- a/src/pkg/oci/push.go +++ b/src/pkg/oci/push.go @@ -7,16 +7,18 @@ package oci import ( "bytes" "encoding/json" + "errors" "fmt" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" + "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" "oras.land/oras-go/v2/content/file" + "oras.land/oras-go/v2/errdef" ) // ConfigPartial is a partial OCI config that is used to create the manifest config. @@ -83,12 +85,13 @@ func (o *OrasRemote) manifestAnnotationsFromMetadata(metadata *types.ZarfMetadat } func (o *OrasRemote) generatePackManifest(src *file.Store, descs []ocispec.Descriptor, configDesc *ocispec.Descriptor, metadata *types.ZarfMetadata) (ocispec.Descriptor, error) { - packOpts := oras.PackOptions{} - packOpts.ConfigDescriptor = configDesc - packOpts.PackImageManifest = true - packOpts.ManifestAnnotations = o.manifestAnnotationsFromMetadata(metadata) + packOpts := oras.PackManifestOptions{ + Layers: descs, + ConfigDescriptor: configDesc, + ManifestAnnotations: o.manifestAnnotationsFromMetadata(metadata), + } - root, err := oras.Pack(o.ctx, src, ocispec.MediaTypeImageManifest, descs, packOpts) + root, err := oras.PackManifest(o.ctx, src, oras.PackManifestVersion1_1_RC4, "", packOpts) if err != nil { return ocispec.Descriptor{}, err } @@ -116,7 +119,7 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag // Get all of the layers in the package var descs []ocispec.Descriptor for name, path := range paths.Files() { - spinner.Updatef("Preparing layer %s", utils.First30last30(name)) + spinner.Updatef("Preparing layer %s", message.First30last30(name)) mediaType := ZarfLayerMediaTypeBlob @@ -153,13 +156,96 @@ func (o *OrasRemote) PublishPackage(pkg *types.ZarfPackage, paths *layout.Packag o.Transport.ProgressBar = message.NewProgressBar(total, fmt.Sprintf("Publishing %s:%s", o.repo.Reference.Repository, o.repo.Reference.Reference)) defer o.Transport.ProgressBar.Stop() - // attempt to push the image manifest - _, err = oras.Copy(ctx, src, root.Digest.String(), o.repo, o.repo.Reference.Reference, copyOpts) + + publishedDesc, err := oras.Copy(ctx, src, root.Digest.String(), o.repo, "", copyOpts) if err != nil { return err } + if err := o.UpdateIndex(o.repo.Reference.Reference, pkg.Build.Architecture, publishedDesc); err != nil { + return err + } o.Transport.ProgressBar.Successf("Published %s [%s]", o.repo.Reference, root.MediaType) return nil } + +// UpdateIndex updates the index for the given package. +func (o *OrasRemote) UpdateIndex(tag string, arch string, publishedDesc ocispec.Descriptor) error { + var index ocispec.Index + + o.repo.Reference.Reference = tag + // since ref has changed, need to reset root + o.root = nil + + platform := &ocispec.Platform{ + OS: MultiOS, + Architecture: arch, + } + + _, err := o.repo.Resolve(o.ctx, o.repo.Reference.Reference) + if err != nil { + if errors.Is(err, errdef.ErrNotFound) { + index = ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Manifests: []ocispec.Descriptor{ + { + MediaType: ocispec.MediaTypeImageManifest, + Digest: publishedDesc.Digest, + Size: publishedDesc.Size, + Platform: platform, + }, + }, + } + return o.pushIndex(&index, tag) + } + return err + } + + desc, rc, err := o.repo.FetchReference(o.ctx, tag) + if err != nil { + return err + } + defer rc.Close() + + b, err := content.ReadAll(rc, desc) + if err != nil { + return err + } + + if err := json.Unmarshal(b, &index); err != nil { + return err + } + + found := false + for idx, m := range index.Manifests { + if m.Platform != nil && m.Platform.Architecture == arch { + index.Manifests[idx].Digest = publishedDesc.Digest + index.Manifests[idx].Size = publishedDesc.Size + index.Manifests[idx].Platform = platform + found = true + break + } + } + if !found { + index.Manifests = append(index.Manifests, ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: publishedDesc.Digest, + Size: publishedDesc.Size, + Platform: platform, + }) + } + + return o.pushIndex(&index, tag) +} + +func (o *OrasRemote) pushIndex(index *ocispec.Index, tag string) error { + indexBytes, err := json.Marshal(index) + if err != nil { + return err + } + indexDesc := content.NewDescriptorFromBytes(ocispec.MediaTypeImageIndex, indexBytes) + return o.repo.Manifests().PushReference(o.ctx, indexDesc, bytes.NewReader(indexBytes), tag) +} diff --git a/src/pkg/oci/utils.go b/src/pkg/oci/utils.go index b15b14be19..149d6d20b7 100644 --- a/src/pkg/oci/utils.go +++ b/src/pkg/oci/utils.go @@ -5,13 +5,11 @@ package oci import ( - "context" "errors" "fmt" "strings" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -19,11 +17,7 @@ import ( ) // ReferenceFromMetadata returns a reference for the given metadata. -// -// prepending the provided prefix -// -// appending the provided suffix to the version -func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata, suffix string) (string, error) { +func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata, build *types.ZarfBuildData) (string, error) { ver := metadata.Version if len(ver) == 0 { return "", errors.New("version is required for publishing") @@ -34,9 +28,12 @@ func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata } registryLocation = strings.TrimPrefix(registryLocation, helpers.OCIURLPrefix) - format := "%s%s:%s-%s" + format := "%s%s:%s" + raw := fmt.Sprintf(format, registryLocation, metadata.Name, ver) - raw := fmt.Sprintf(format, registryLocation, metadata.Name, ver, suffix) + if build != nil && build.Flavor != "" { + raw = fmt.Sprintf("%s-%s", raw, build.Flavor) + } message.Debug("Raw OCI reference from metadata:", raw) @@ -48,29 +45,6 @@ func ReferenceFromMetadata(registryLocation string, metadata *types.ZarfMetadata return ref.String(), nil } -// printLayerSkipped prints a debug message when a layer has been successfully skipped. -func (o *OrasRemote) printLayerSkipped(_ context.Context, desc ocispec.Descriptor) error { - return o.printLayer(desc, "skipped") -} - -// printLayerCopied prints a debug message when a layer has been successfully copied to/from a registry. -func (o *OrasRemote) printLayerCopied(_ context.Context, desc ocispec.Descriptor) error { - return o.printLayer(desc, "copied") -} - -// printLayer prints a debug message when a layer has been successfully published/pulled to/from a registry. -func (o *OrasRemote) printLayer(desc ocispec.Descriptor, suffix string) error { - title := desc.Annotations[ocispec.AnnotationTitle] - var layerInfo string - if title != "" { - layerInfo = fmt.Sprintf("%s %s", desc.Digest.Encoded()[:12], utils.First30last30(title)) - } else { - layerInfo = fmt.Sprintf("%s [%s]", desc.Digest.Encoded()[:12], desc.MediaType) - } - message.Debugf("%s (%s)", layerInfo, suffix) - return nil -} - // IsEmptyDescriptor returns true if the given descriptor is empty. func IsEmptyDescriptor(desc ocispec.Descriptor) bool { return desc.Digest == "" && desc.Size == 0 @@ -102,7 +76,7 @@ func RemoveDuplicateDescriptors(descriptors []ocispec.Descriptor) []ocispec.Desc return list } -// GetInitPackageURL returns the URL for the init package for the given architecture and version. -func GetInitPackageURL(arch, version string) string { - return fmt.Sprintf("ghcr.io/defenseunicorns/packages/init:%s-%s", version, arch) +// GetInitPackageURL returns the URL for the init package for the given version. +func GetInitPackageURL(version string) string { + return fmt.Sprintf("ghcr.io/defenseunicorns/packages/init:%s", version) } diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 77ed76c6e3..e1cbfb0a20 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -28,7 +28,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -38,7 +37,6 @@ import ( type Packager struct { cfg *types.PackagerConfig cluster *cluster.Cluster - remote *oci.OrasRemote layout *layout.PackagePaths arch string warnings []string @@ -334,7 +332,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { } spinner.Updatef("Wrote %s to %s", p.layout.Base, destinationTarball) - f, err := os.Stat(destinationTarball) + fi, err := os.Stat(destinationTarball) if err != nil { return fmt.Errorf("unable to read the package archive: %w", err) } @@ -343,7 +341,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { chunkSize := p.cfg.CreateOpts.MaxPackageSizeMB * 1000 * 1000 // If a chunk size was specified and the package is larger than the chunk size, split it into chunks. - if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && f.Size() > int64(chunkSize) { + if p.cfg.CreateOpts.MaxPackageSizeMB > 0 && fi.Size() > int64(chunkSize) { spinner.Updatef("Package is larger than %dMB, splitting into multiple files", p.cfg.CreateOpts.MaxPackageSizeMB) chunks, sha256sum, err := utils.SplitFile(destinationTarball, chunkSize) if err != nil { @@ -361,7 +359,7 @@ func (p *Packager) archivePackage(destinationTarball string) error { // Marshal the data into a json file. jsonData, err := json.Marshal(types.ZarfSplitPackageData{ Count: len(chunks), - Bytes: f.Size(), + Bytes: fi.Size(), Sha256Sum: sha256sum, }) if err != nil { @@ -385,16 +383,6 @@ func (p *Packager) archivePackage(destinationTarball string) error { return nil } -// setOCIRemote sets the remote OCI client for the package. -func (p *Packager) setOCIRemote(url string) error { - remote, err := oci.NewOrasRemote(url) - if err != nil { - return err - } - p.remote = remote - return nil -} - func (p *Packager) signPackage(signingKeyPath, signingKeyPassword string) error { p.layout = p.layout.AddSignature(signingKeyPath) passwordFunc := func(_ bool) ([]byte, error) { diff --git a/src/pkg/packager/components.go b/src/pkg/packager/components.go index 45b9cf5ae3..d393eb3e59 100644 --- a/src/pkg/packager/components.go +++ b/src/pkg/packager/components.go @@ -5,211 +5,182 @@ package packager import ( - "fmt" + "path" + "slices" + "strings" - "github.com/AlecAivazis/survey/v2" - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/k8s" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/pkg/interactive" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" - "github.com/pterm/pterm" ) -func (p *Packager) getValidComponents() []types.ZarfComponent { - var validComponentsList []types.ZarfComponent - var orderedKeys []string - var choiceComponents []string +type selectState int - componentGroups := make(map[string][]types.ZarfComponent) +const ( + unknown selectState = iota + included + excluded +) - // The component list is comma-delimited list - requestedNames := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) +func (p *Packager) getSelectedComponents() []types.ZarfComponent { + var selectedComponents []types.ZarfComponent + groupedComponents := map[string][]types.ZarfComponent{} + orderedComponentGroups := []string{} - // Break up components into choice groups + // Group the components by Name and Group while maintaining order for _, component := range p.cfg.Pkg.Components { - matchFn := func(a, b string) bool { return a == b } - key := component.Group - // If not a choice group, then use the component name as the key - if key == "" { - key = component.Name - } else { - // Otherwise, add the component name to the choice group list for later validation - choiceComponents = helpers.MergeSlices(choiceComponents, []string{component.Name}, matchFn) + groupKey := component.Name + if component.Group != "" { + groupKey = component.Group } - // Preserve component order - orderedKeys = helpers.MergeSlices(orderedKeys, []string{key}, matchFn) + if !slices.Contains(orderedComponentGroups, groupKey) { + orderedComponentGroups = append(orderedComponentGroups, groupKey) + } - // Append the component to the list of components in the group - componentGroups[key] = append(componentGroups[key], component) + groupedComponents[groupKey] = append(groupedComponents[groupKey], component) } - // Loop through each component group in original order and handle required, requested or user confirmation - for _, key := range orderedKeys { - - componentGroup := componentGroups[key] + // Split the --components list as a comma-delimited list + requestedComponents := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) + isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" + + if isPartial { + matchedRequests := map[string]bool{} + + // NOTE: This does not use forIncludedComponents as it takes group, default and required status into account. + for _, groupKey := range orderedComponentGroups { + var groupDefault *types.ZarfComponent + var groupSelected *types.ZarfComponent + + for _, component := range groupedComponents[groupKey] { + // Ensure we have a local version of the component to point to (otherwise the pointer might change on us) + component := component + + selectState, matchedRequest := includedOrExcluded(component, requestedComponents) + + if !component.Required { + if selectState == excluded { + // If the component was explicitly excluded, record the match and continue + matchedRequests[matchedRequest] = true + continue + } else if selectState == unknown && component.Default && groupDefault == nil { + // If the component is default but not included or excluded, remember the default + groupDefault = &component + } + } else { + // Force the selectState to included for Required components + selectState = included + } - // Choice groups are handled differently for user confirmation - userChoicePrompt := len(componentGroup) > 1 + if selectState == included { + // If the component was explicitly included, record the match + matchedRequests[matchedRequest] = true - // Loop through the components in the group - for _, component := range componentGroup { - // First check if the component is required or requested via CLI flag - requested := p.isRequiredOrRequested(component, requestedNames) + // Then check for already selected groups + if groupSelected != nil { + message.Fatalf(nil, lang.PkgDeployErrMultipleComponentsSameGroup, groupSelected.Name, component.Name, component.Group) + } - // If the user has not requested this component via CLI flag, then prompt them if not a choice group - if !requested && !userChoicePrompt { - requested = p.confirmOptionalComponent(component) + // Then append to the final list + selectedComponents = append(selectedComponents, component) + groupSelected = &component + } } - if requested { - // Mark deployment as appliance mode if this is an init config and the k3s component is enabled - if component.Name == k8s.DistroIsK3s && p.isInitConfig() { - p.cfg.InitOpts.ApplianceMode = true + // If nothing was selected from a group, handle the default + if groupSelected == nil && groupDefault != nil { + selectedComponents = append(selectedComponents, *groupDefault) + } else if len(groupedComponents[groupKey]) > 1 && groupSelected == nil && groupDefault == nil { + // If no default component was found, give up + componentNames := []string{} + for _, component := range groupedComponents[groupKey] { + componentNames = append(componentNames, component.Name) } - // Add the component to the list of valid components - validComponentsList = append(validComponentsList, component) - // Ensure that the component is not requested again if in a choice group - userChoicePrompt = false - // Exit the inner loop on a match since groups should only have one requested component - break + message.Fatalf(nil, lang.PkgDeployErrNoDefaultOrSelection, strings.Join(componentNames, ",")) } } - // If the user has requested a choice group, then prompt them - if userChoicePrompt { - selectedComponent := p.confirmChoiceGroup(componentGroup) - validComponentsList = append(validComponentsList, selectedComponent) - } - } - - // Ensure all user requested components are valid - if err := p.validateRequests(validComponentsList, requestedNames, choiceComponents); err != nil { - message.Fatalf(err, "Invalid component argument, %s", err) - } - - return validComponentsList -} - -// Match on the first requested component that is not in the list of valid components and return the component name. -func (p *Packager) validateRequests(validComponentsList []types.ZarfComponent, requestedComponentNames, choiceComponents []string) error { - // Loop through each requested component names - for _, componentName := range requestedComponentNames { - found := false - // Match on the first requested component that is a valid component - for _, component := range validComponentsList { - if component.Name == componentName { - found = true - break + // Check that we have matched against all requests + for _, requestedComponent := range requestedComponents { + if _, ok := matchedRequests[requestedComponent]; !ok { + message.Fatalf(nil, lang.PkgDeployErrNoCompatibleComponentsForSelection, requestedComponent) } } - - // If the requested component was not found, then return an error - if !found { - // If the requested component is in a choice group, then warn the user they must choose only one - for _, component := range choiceComponents { - if component == componentName { - return fmt.Errorf("component %s is part of a group of components and only one may be chosen", componentName) + } else { + for _, groupKey := range orderedComponentGroups { + if len(groupedComponents[groupKey]) > 1 { + component := interactive.SelectChoiceGroup(groupedComponents[groupKey]) + selectedComponents = append(selectedComponents, component) + } else { + component := groupedComponents[groupKey][0] + + if component.Required { + selectedComponents = append(selectedComponents, component) + } else if selected := interactive.SelectOptionalComponent(component); selected { + selectedComponents = append(selectedComponents, component) } } - // Otherwise, return an error a general error - return fmt.Errorf("unable to find component %s", componentName) } } - return nil + return selectedComponents } -func (p *Packager) isRequiredOrRequested(component types.ZarfComponent, requestedComponentNames []string) bool { - // If the component is required, then just return true - if component.Required { - return true - } - - // Otherwise,check if this is one of the components that has been requested - if len(requestedComponentNames) > 0 || config.CommonOptions.Confirm { - for _, requestedComponent := range requestedComponentNames { - // If the component name matches one of the requested components, then return true - if requestedComponent == component.Name { - return true - } - } - } +func (p *Packager) forIncludedComponents(onIncluded func(types.ZarfComponent) error) error { + requestedComponents := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) + isPartial := len(requestedComponents) > 0 && requestedComponents[0] != "" - // All other cases, return false - return false -} + for _, component := range p.cfg.Pkg.Components { + selectState := unknown -// Confirm optional component. -func (p *Packager) confirmOptionalComponent(component types.ZarfComponent) (confirmComponent bool) { - // Confirm flag passed, just use defaults - if config.CommonOptions.Confirm { - return component.Default - } + if isPartial { + selectState, _ = includedOrExcluded(component, requestedComponents) - message.HorizontalRule() + if selectState == excluded { + continue + } + } else { + selectState = included + } - displayComponent := component - displayComponent.Description = "" - utils.ColorPrintYAML(displayComponent, nil, false) - if component.Description != "" { - message.Question(component.Description) + if selectState == included { + if err := onIncluded(component); err != nil { + return err + } + } } - // Since no requested components were provided, prompt the user - prompt := &survey.Confirm{ - Message: fmt.Sprintf("Deploy the %s component?", component.Name), - Default: component.Default, - } - if err := survey.AskOne(prompt, &confirmComponent); err != nil { - message.Fatalf(nil, "Confirm selection canceled: %s", err.Error()) - } - return confirmComponent + return nil } -func (p *Packager) confirmChoiceGroup(componentGroup []types.ZarfComponent) types.ZarfComponent { - // Confirm flag passed, just use defaults - if config.CommonOptions.Confirm { - var componentNames []string - for _, component := range componentGroup { - // If the component is default, then return it - if component.Default { - return component +func includedOrExcluded(component types.ZarfComponent, requestedComponentNames []string) (selectState, string) { + // Check if the component has a leading dash indicating it should be excluded - this is done first so that exclusions precede inclusions + for _, requestedComponent := range requestedComponentNames { + if strings.HasPrefix(requestedComponent, "-") { + // If the component glob matches one of the requested components, then return true + // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) + if matched, _ := path.Match(strings.TrimPrefix(requestedComponent, "-"), component.Name); matched { + return excluded, requestedComponent } - // Add each component name to the list - componentNames = append(componentNames, component.Name) } - // If no default component was found, give up - message.Fatalf(nil, "You must specify at least one component from the group %#v when using the --confirm flag.", componentNames) - } - - message.HorizontalRule() - - var chosen int - var options []string - - for _, component := range componentGroup { - text := fmt.Sprintf("Name: %s\n Description: %s\n", component.Name, component.Description) - options = append(options, text) } - - prompt := &survey.Select{ - Message: "Select a component to deploy:", - Options: options, - } - - pterm.Println() - - if err := survey.AskOne(prompt, &chosen); err != nil { - message.Fatalf(nil, "Component selection canceled: %s", err.Error()) + // Check if the component matches a glob pattern and should be included + for _, requestedComponent := range requestedComponentNames { + // If the component glob matches one of the requested components, then return true + // This supports globbing with "path" in order to have the same behavior across OSes (if we ever allow namespaced components with /) + if matched, _ := path.Match(requestedComponent, component.Name); matched { + return included, requestedComponent + } } - return componentGroup[chosen] + // All other cases we don't know if we should include or exclude yet + return unknown, "" } -func (p *Packager) requiresCluster(component types.ZarfComponent) bool { +func requiresCluster(component types.ZarfComponent) bool { hasImages := len(component.Images) > 0 hasCharts := len(component.Charts) > 0 hasManifests := len(component.Manifests) > 0 diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 660659c5a2..6bbbf22fff 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -17,7 +17,7 @@ func (p *Packager) composeComponents() error { pkgVars := p.cfg.Pkg.Variables pkgConsts := p.cfg.Pkg.Constants - for _, component := range p.cfg.Pkg.Components { + for i, component := range p.cfg.Pkg.Components { arch := p.arch // filter by architecture if !composer.CompatibleComponent(component, arch, p.cfg.CreateOpts.Flavor) { @@ -29,7 +29,7 @@ func (p *Packager) composeComponents() error { component.Only.Flavor = "" // build the import chain - chain, err := composer.NewImportChain(component, arch, p.cfg.CreateOpts.Flavor) + chain, err := composer.NewImportChain(component, i, p.cfg.Pkg.Metadata.Name, arch, p.cfg.CreateOpts.Flavor) if err != nil { return err } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f38eeac49d..f46235f212 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -22,17 +22,49 @@ import ( type Node struct { types.ZarfComponent + index int + vars []types.ZarfPackageVariable consts []types.ZarfPackageConstant - relativeToHead string + relativeToHead string + originalPackageName string prev *Node next *Node } +// GetIndex returns the .components index location for this node's source `zarf.yaml` +func (n *Node) GetIndex() int { + return n.index +} + +// GetOriginalPackageName returns the .metadata.name of the zarf package the component originated from +func (n *Node) GetOriginalPackageName() string { + return n.originalPackageName +} + +// ImportLocation gets the path from the base zarf file to the imported zarf file +func (n *Node) ImportLocation() string { + if n.prev != nil { + if n.prev.ZarfComponent.Import.URL != "" { + return n.prev.ZarfComponent.Import.URL + } + } + return n.relativeToHead +} + +// Next returns next node in the chain +func (n *Node) Next() *Node { + return n.next +} + +// Prev returns previous node in the chain +func (n *Node) Prev() *Node { + return n.prev +} + // ImportName returns the name of the component to import -// // If the component import has a ComponentName defined, that will be used // otherwise the name of the component will be used func (n *Node) ImportName() string { @@ -51,14 +83,27 @@ type ImportChain struct { remote *oci.OrasRemote } -func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { +// Head returns the first node in the import chain +func (ic *ImportChain) Head() *Node { + return ic.head +} + +// Tail returns the last node in the import chain +func (ic *ImportChain) Tail() *Node { + return ic.tail +} + +func (ic *ImportChain) append(c types.ZarfComponent, index int, originalPackageName string, + relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { node := &Node{ - ZarfComponent: c, - relativeToHead: relativeToHead, - vars: vars, - consts: consts, - prev: nil, - next: nil, + ZarfComponent: c, + index: index, + originalPackageName: originalPackageName, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, } if ic.head == nil { ic.head = node @@ -72,14 +117,14 @@ func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars } // NewImportChain creates a new import chain from a component -func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain, error) { +// Returning the chain on error so we can have additional information to use during lint +func NewImportChain(head types.ZarfComponent, index int, originalPackageName, arch, flavor string) (*ImportChain, error) { + ic := &ImportChain{} if arch == "" { - return nil, fmt.Errorf("cannot build import chain: architecture must be provided") + return ic, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic := &ImportChain{} - - ic.append(head, ".", nil, nil) + ic.append(head, index, originalPackageName, ".", nil, nil) history := []string{} @@ -110,9 +155,11 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain var pkg types.ZarfPackage + var relativeToHead string + var importURL string if isLocal { history = append(history, node.Import.Path) - relativeToHead := filepath.Join(history...) + relativeToHead = filepath.Join(history...) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small @@ -129,6 +176,7 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain return ic, err } } else if isRemote { + importURL = node.Import.URL remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err @@ -141,26 +189,34 @@ func NewImportChain(head types.ZarfComponent, arch, flavor string) (*ImportChain name := node.ImportName() - found := helpers.Filter(pkg.Components, func(c types.ZarfComponent) bool { - matchesName := c.Name == name - return matchesName && CompatibleComponent(c, arch, flavor) - }) + // 'found' and 'index' are parallel slices. Each element in found[x] corresponds to pkg[index[x]] + // found[0] and pkg[index[0]] would be the same componenet for example + found := []types.ZarfComponent{} + index := []int{} + for i, component := range pkg.Components { + if component.Name == name && CompatibleComponent(component, arch, flavor) { + found = append(found, component) + index = append(index, i) + } + } if len(found) == 0 { + componentNotFound := "component %q not found in %q" if isLocal { - return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) + return ic, fmt.Errorf(componentNotFound, name, relativeToHead) } else if isRemote { - return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) + return ic, fmt.Errorf(componentNotFound, name, importURL) } } else if len(found) > 1 { + multipleComponentsFound := "multiple components named %q found in %q satisfying %q" if isLocal { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) + return ic, fmt.Errorf(multipleComponentsFound, name, relativeToHead, arch) } else if isRemote { - return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) + return ic, fmt.Errorf(multipleComponentsFound, name, importURL, arch) } } - ic.append(found[0], filepath.Join(history...), pkg.Variables, pkg.Constants) + ic.append(found[0], index[0], pkg.Metadata.Name, relativeToHead, pkg.Variables, pkg.Constants) node = node.next } return ic, nil diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index f255803fde..d80f3633c3 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -43,14 +43,14 @@ func TestNewImportChain(t *testing.T) { expectedErrorMessage: "detected circular import chain", }, } - + testPackageName := "test-package" for _, testCase := range testCases { testCase := testCase t.Run(testCase.name, func(t *testing.T) { t.Parallel() - _, err := NewImportChain(testCase.head, testCase.arch, testCase.flavor) + _, err := NewImportChain(testCase.head, 0, testPackageName, testCase.arch, testCase.flavor) require.Contains(t, err.Error(), testCase.expectedErrorMessage) }) } @@ -441,17 +441,18 @@ func TestMerging(t *testing.T) { func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { ic = &ImportChain{} + testPackageName := "test-package" if len(components) == 0 { return ic } - ic.append(components[0], ".", nil, nil) + ic.append(components[0], 0, testPackageName, ".", nil, nil) history := []string{} for idx := 1; idx < len(components); idx++ { history = append(history, components[idx-1].Import.Path) - ic.append(components[idx], filepath.Join(history...), nil, nil) + ic.append(components[idx], idx, testPackageName, filepath.Join(history...), nil, nil) } return ic diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 5a7b4637ed..0477d04b97 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -27,10 +27,14 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { return ic.remote, nil } var err error - ic.remote, err = oci.NewOrasRemote(url) + ic.remote, err = oci.NewOrasRemote(url, oci.WithSkeletonArch()) if err != nil { return nil, err } + _, err = ic.remote.ResolveRoot() + if err != nil { + return nil, fmt.Errorf("published skeleton package for %q does not exist: %w", url, err) + } return ic.remote, nil } diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index f39947bffc..02f13b240b 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -5,102 +5,29 @@ package packager import ( - "errors" "fmt" "os" - "path/filepath" - "strconv" - "strings" - "time" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/internal/packager/git" - "github.com/defenseunicorns/zarf/src/internal/packager/helm" - "github.com/defenseunicorns/zarf/src/internal/packager/images" - "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" - "github.com/defenseunicorns/zarf/src/internal/packager/sbom" "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/transform" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/defenseunicorns/zarf/src/types" - "github.com/go-git/go-git/v5/plumbing" - "github.com/mholt/archiver/v3" ) // Create generates a Zarf package tarball for a given PackageConfig and optional base directory. func (p *Packager) Create() (err error) { - if err = p.readZarfYAML(filepath.Join(p.cfg.CreateOpts.BaseDir, layout.ZarfYAML)); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - - // Load the images and repos from the 'reference' package - if err := p.loadDifferentialData(); err != nil { - return err - } - cwd, err := os.Getwd() if err != nil { return err } - if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { - return fmt.Errorf("unable to access directory '%s': %w", p.cfg.CreateOpts.BaseDir, err) - } - message.Note(fmt.Sprintf("Using build directory %s", p.cfg.CreateOpts.BaseDir)) - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - // Compose components into a single zarf.yaml file - if err := p.composeComponents(); err != nil { + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { return err } - // After components are composed, template the active package. - if err := p.fillActiveTemplate(); err != nil { - return fmt.Errorf("unable to fill values in template: %s", err.Error()) - } - - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, p.arch) - if err != nil { - return err - } - err = p.setOCIRemote(ref) - if err != nil { - return err - } - } - - // After templates are filled process any create extensions - if err := p.processExtensions(); err != nil { + if err := p.load(); err != nil { return err } - // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { - // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building - // If the package versions are the same return an error - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { - return errors.New(lang.PkgCreateErrDifferentialSameVersion) - } - if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { - return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") - } - - // Handle any potential differential images/repos before going forward - if err := p.removeCopiesFromDifferentialPackage(); err != nil { - return err - } - } - // Perform early package validation. if err := validate.Run(p.cfg.Pkg); err != nil { return fmt.Errorf("unable to validate package: %w", err) @@ -110,632 +37,14 @@ func (p *Packager) Create() (err error) { return fmt.Errorf("package creation canceled") } - componentSBOMs := map[string]*layout.ComponentSBOM{} - var combinedImageList []transform.Image - for idx, component := range p.cfg.Pkg.Components { - onCreate := component.Actions.OnCreate - onFailure := func() { - if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { - message.Debugf("unable to run component failure action: %s", err.Error()) - } - } - isSkeleton := false - if err := p.addComponent(idx, component, isSkeleton); err != nil { - onFailure() - return fmt.Errorf("unable to add component %q: %w", component.Name, err) - } - componentSBOM, err := p.getFilesToSBOM(component) - if err != nil { - onFailure() - return fmt.Errorf("unable to create component SBOM: %w", err) - } - - if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { - onFailure() - return fmt.Errorf("unable to run component success action: %w", err) - } - - if componentSBOM != nil && len(componentSBOM.Files) > 0 { - componentSBOMs[component.Name] = componentSBOM - } - - // Combine all component images into a single entry for efficient layer reuse. - for _, src := range component.Images { - refInfo, err := transform.ParseImageRef(src) - if err != nil { - return fmt.Errorf("failed to create ref for image %s: %w", src, err) - } - combinedImageList = append(combinedImageList, refInfo) - } - } - - imageList := helpers.Unique(combinedImageList) - var sbomImageList []transform.Image - - // Images are handled separately from other component assets. - if len(imageList) > 0 { - message.HeaderInfof("📦 PACKAGE IMAGES") - - p.layout = p.layout.AddImages() - - var pulled []images.ImgInfo - - doPull := func() error { - imgConfig := images.ImageConfig{ - ImagesPath: p.layout.Images.Base, - ImageList: imageList, - Insecure: config.CommonOptions.Insecure, - Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, - RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, - } - - pulled, err = imgConfig.PullAll() - return err - } - - if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after 3 attempts: %w", err) - } - - for _, imgInfo := range pulled { - if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { - return err - } - if imgInfo.HasImageLayers { - sbomImageList = append(sbomImageList, imgInfo.RefInfo) - } - } - } - - // Ignore SBOM creation if there the flag is set. - if p.cfg.CreateOpts.SkipSBOM { - message.Debug("Skipping image SBOM processing per --skip-sbom flag") - } else { - p.layout = p.layout.AddSBOMs() - if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { - return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) - } - } - - // Process the component directories into compressed tarballs - // NOTE: This is purposefully being done after the SBOM cataloging - for _, component := range p.cfg.Pkg.Components { - // Make the component a tar archive - if err := p.layout.Components.Archive(component, true); err != nil { - return fmt.Errorf("unable to archive component: %s", err.Error()) - } - } - - // Calculate all the checksums - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for the package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - // Save the transformed config. - if err := p.writeYaml(); err != nil { - return fmt.Errorf("unable to write zarf.yaml: %w", err) - } - - // cd back - if err := os.Chdir(cwd); err != nil { + if err := p.assemble(); err != nil { return err } - // Sign the config file if a key was provided - if p.cfg.CreateOpts.SigningKeyPath != "" { - if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { - return err - } - } - - if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { - err := p.remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) - if err != nil { - return fmt.Errorf("unable to publish package: %w", err) - } - message.HorizontalRule() - flags := "" - if config.CommonOptions.Insecure { - flags = "--insecure" - } - message.Title("To inspect/deploy/pull:", "") - message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) - message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) - message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+p.remote.Repo().Reference.String(), flags) - } else { - // Use the output path if the user specified it. - packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) - - // Try to remove the package if it already exists. - _ = os.Remove(packageName) - - // Create the package tarball. - if err := p.archivePackage(packageName); err != nil { - return fmt.Errorf("unable to archive package: %w", err) - } - } - - // Output the SBOM files into a directory if specified. - if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { - outputSBOM := p.cfg.CreateOpts.SBOMOutputDir - var sbomDir string - if err := p.layout.SBOMs.Unarchive(); err != nil { - return fmt.Errorf("unable to unarchive SBOMs: %w", err) - } - sbomDir = p.layout.SBOMs.Path - - if outputSBOM != "" { - out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) - if err != nil { - return err - } - sbomDir = out - } - - if p.cfg.CreateOpts.ViewSBOM { - sbom.ViewSBOMFiles(sbomDir) - } - } - - return nil -} - -func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { - componentPaths, err := p.layout.Components.Create(component) - if err != nil { - return nil, err - } - // Create an struct to hold the SBOM information for this component. - componentSBOM := &layout.ComponentSBOM{ - Files: []string{}, - Component: componentPaths, - } - - appendSBOMFiles := func(path string) { - if utils.IsDir(path) { - files, _ := utils.RecursiveFileList(path, nil, false) - componentSBOM.Files = append(componentSBOM.Files, files...) - } else { - componentSBOM.Files = append(componentSBOM.Files, path) - } - } - - for filesIdx, file := range component.Files { - path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - appendSBOMFiles(path) - } - - for dataIdx, data := range component.DataInjections { - path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - - appendSBOMFiles(path) - } - - return componentSBOM, nil -} - -func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkeleton bool) error { - message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) - - componentPaths, err := p.layout.Components.Create(component) - if err != nil { + // cd back for output + if err := os.Chdir(cwd); err != nil { return err } - if isSkeleton && component.DeprecatedCosignKeyPath != "" { - dst := filepath.Join(componentPaths.Base, "cosign.pub") - err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) - if err != nil { - return err - } - p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" - } - - // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. - if isSkeleton { - component.Actions.OnCreate.Defaults.Dir = "" - resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { - for idx := range actions { - actions[idx].Dir = nil - } - return actions - } - component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) - component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) - component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) - component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) - } - - onCreate := component.Actions.OnCreate - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { - return fmt.Errorf("unable to run component before action: %w", err) - } - } - - // If any helm charts are defined, process them. - for chartIdx, chart := range component.Charts { - - helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) - - if isSkeleton { - if chart.LocalPath != "" { - rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - err := utils.CreatePathAndCopy(chart.LocalPath, dst) - if err != nil { - return err - } - - p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel - } - - for valuesIdx, path := range chart.ValuesFiles { - if helpers.IsURL(path) { - continue - } - - rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) - p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel - - if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { - return fmt.Errorf("unable to copy chart values file %s: %w", path, err) - } - } - } else { - err := helmCfg.PackageChart(componentPaths.Charts) - if err != nil { - return err - } - } - } - - for filesIdx, file := range component.Files { - message.Debugf("Loading %#v", file) - - rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - dst := filepath.Join(componentPaths.Base, rel) - destinationDir := filepath.Dir(dst) - - if helpers.IsURL(file.Source) { - if isSkeleton { - continue - } - - if file.ExtractPath != "" { - - // get the compressedFileName from the source - compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) - if err != nil { - return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) - } - - compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) - - // If the file is an archive, download it to the componentPath.Temp - if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - - err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) - if err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) - } - - } else { - if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) - } - } - - } else { - if file.ExtractPath != "" { - if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { - return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) - } - } - - } - - if file.ExtractPath != "" { - // Make sure dst reflects the actual file or directory. - updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) - if updatedExtractedFileOrDir != dst { - if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { - return fmt.Errorf(lang.ErrWritingFile, dst, err) - } - } - } - - if isSkeleton { - // Change the source to the new relative source directory (any remote files will have been skipped above) - p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel - // Remove the extractPath from a skeleton since it will already extract it - p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" - } - - // Abort packaging on invalid shasum (if one is specified). - if file.Shasum != "" { - if err := utils.SHAsMatch(dst, file.Shasum); err != nil { - return err - } - } - - if file.Executable || utils.IsDir(dst) { - _ = os.Chmod(dst, 0700) - } else { - _ = os.Chmod(dst, 0600) - } - } - - if len(component.DataInjections) > 0 { - spinner := message.NewProgressSpinner("Loading data injections") - defer spinner.Stop() - - for dataIdx, data := range component.DataInjections { - spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) - - rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) - dst := filepath.Join(componentPaths.Base, rel) - - if helpers.IsURL(data.Source) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { - return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) - } - if isSkeleton { - p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel - } - } - } - spinner.Success() - } - - if len(component.Manifests) > 0 { - // Get the proper count of total manifests to add. - manifestCount := 0 - - for _, manifest := range component.Manifests { - manifestCount += len(manifest.Files) - manifestCount += len(manifest.Kustomizations) - } - - spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) - defer spinner.Stop() - - // Iterate over all manifests. - for manifestIdx, manifest := range component.Manifests { - for fileIdx, path := range manifest.Files { - rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) - dst := filepath.Join(componentPaths.Base, rel) - - // Copy manifests without any processing. - spinner.Updatef("Copying manifest %s", path) - if helpers.IsURL(path) { - if isSkeleton { - continue - } - if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, path, err.Error()) - } - } else { - if err := utils.CreatePathAndCopy(path, dst); err != nil { - return fmt.Errorf("unable to copy manifest %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel - } - } - } - - for kustomizeIdx, path := range manifest.Kustomizations { - // Generate manifests from kustomizations and place in the package. - spinner.Updatef("Building kustomization for %s", path) - - kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) - rel := filepath.Join(layout.ManifestsDir, kname) - dst := filepath.Join(componentPaths.Base, rel) - - if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { - return fmt.Errorf("unable to build kustomization %s: %w", path, err) - } - if isSkeleton { - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) - } - } - if isSkeleton { - // remove kustomizations - p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil - } - } - spinner.Success() - } - - // Load all specified git repos. - if len(component.Repos) > 0 && !isSkeleton { - spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) - defer spinner.Stop() - - for _, url := range component.Repos { - // Pull all the references if there is no `@` in the string. - gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) - if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { - return fmt.Errorf("unable to pull git repo %s: %w", url, err) - } - } - spinner.Success() - } - - if !isSkeleton { - if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { - return fmt.Errorf("unable to run component after action: %w", err) - } - } - - return nil -} - -// generateChecksum walks through all of the files starting at the base path and generates a checksum file. -// Each file within the basePath represents a layer within the Zarf package. -// generateChecksum returns a SHA256 checksum of the checksums.txt file. -func (p *Packager) generatePackageChecksums() (string, error) { - var checksumsData string - - // Loop over the "loaded" files - for rel, abs := range p.layout.Files() { - if rel == layout.ZarfYAML || rel == layout.Checksums { - continue - } - - sum, err := utils.GetSHA256OfFile(abs) - if err != nil { - return "", err - } - checksumsData += fmt.Sprintf("%s %s\n", sum, rel) - } - - // Create the checksums file - checksumsFilePath := p.layout.Checksums - if err := utils.WriteFile(checksumsFilePath, []byte(checksumsData)); err != nil { - return "", err - } - - // Calculate the checksum of the checksum file - return utils.GetSHA256OfFile(checksumsFilePath) -} - -// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package -func (p *Packager) loadDifferentialData() error { - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Save the fact that this is a differential build into the build data of the package - p.cfg.Pkg.Build.Differential = true - - tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) - defer os.RemoveAll(tmpDir) - - // Load the package spec of the package we're using as a 'reference' for the differential build - if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { - err := p.setOCIRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) - if err != nil { - return err - } - pkg, err := p.remote.FetchZarfYAML() - if err != nil { - return err - } - err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) - if err != nil { - return err - } - } else { - if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { - return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) - } - } - - var differentialZarfConfig types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { - return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) - } - - // Generate a map of all the images and repos that are included in the provided package - allIncludedImagesMap := map[string]bool{} - allIncludedReposMap := map[string]bool{} - for _, component := range differentialZarfConfig.Components { - for _, image := range component.Images { - allIncludedImagesMap[image] = true - } - for _, repo := range component.Repos { - allIncludedReposMap[repo] = true - } - } - - p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap - p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap - p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - - return nil -} - -// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package -func (p *Packager) removeCopiesFromDifferentialPackage() error { - // If a differential build was not requested, continue on as normal - if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { - return nil - } - - // Loop through all of the components to determine if any of them are using already included images or repos - componentMap := make(map[int]types.ZarfComponent) - for idx, component := range p.cfg.Pkg.Components { - newImageList := []string{} - newRepoList := []string{} - // Generate a list of all unique images for this component - for _, img := range component.Images { - // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package - imgRef, err := transform.ParseImageRef(img) - if err != nil { - return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) - } - - // Only include new images or images that have a commonly overwritten tag - imgTag := imgRef.TagOrDigest - useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" - if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { - newImageList = append(newImageList, img) - } else { - message.Debugf("Image %s is already included in the differential package", img) - } - } - - // Generate a list of all unique repos for this component - for _, repoURL := range component.Repos { - // Split the remote url and the zarf reference - _, refPlain, err := transform.GitURLSplitRef(repoURL) - if err != nil { - return err - } - - var ref plumbing.ReferenceName - // Parse the ref from the git URL. - if refPlain != "" { - ref = git.ParseRef(refPlain) - } - - // Only include new repos or repos that were not referenced by a specific commit sha or tag - useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) - if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { - newRepoList = append(newRepoList, repoURL) - } else { - message.Debugf("Repo %s is already included in the differential package", repoURL) - } - } - - // Update the component with the unique lists of repos and images - component.Images = newImageList - component.Repos = newRepoList - componentMap[idx] = component - } - - // Update the package with the new component list - for idx, component := range componentMap { - p.cfg.Pkg.Components[idx] = component - } - - return nil + return p.output() } diff --git a/src/pkg/packager/create_stages.go b/src/pkg/packager/create_stages.go new file mode 100644 index 0000000000..2ae7f06ad9 --- /dev/null +++ b/src/pkg/packager/create_stages.go @@ -0,0 +1,762 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" + "github.com/defenseunicorns/zarf/src/internal/packager/git" + "github.com/defenseunicorns/zarf/src/internal/packager/helm" + "github.com/defenseunicorns/zarf/src/internal/packager/images" + "github.com/defenseunicorns/zarf/src/internal/packager/kustomize" + "github.com/defenseunicorns/zarf/src/internal/packager/sbom" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/transform" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" + "github.com/go-git/go-git/v5/plumbing" + "github.com/mholt/archiver/v3" +) + +func (p *Packager) cdToBaseDir(base string, cwd string) error { + if err := os.Chdir(base); err != nil { + return fmt.Errorf("unable to access directory %q: %w", base, err) + } + message.Note(fmt.Sprintf("Using build directory %s", base)) + + // differentials are relative to the current working directory + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath = filepath.Join(cwd, p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + } + return nil +} + +func (p *Packager) load() error { + if err := p.readZarfYAML(layout.ZarfYAML); err != nil { + return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) + } + if p.isInitConfig() { + p.cfg.Pkg.Metadata.Version = config.CLIVersion + } + + // Compose components into a single zarf.yaml file + if err := p.composeComponents(); err != nil { + return err + } + + if p.cfg.CreateOpts.IsSkeleton { + return nil + } + + // After components are composed, template the active package. + if err := p.fillActiveTemplate(); err != nil { + return fmt.Errorf("unable to fill values in template: %s", err.Error()) + } + + // After templates are filled process any create extensions + if err := p.processExtensions(); err != nil { + return err + } + + // After we have a full zarf.yaml remove unnecessary repos and images if we are building a differential package + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" { + // Load the images and repos from the 'reference' package + if err := p.loadDifferentialData(); err != nil { + return err + } + // Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building + // If the package versions are the same return an error + if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version { + return errors.New(lang.PkgCreateErrDifferentialSameVersion) + } + if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" { + return fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set") + } + + // Handle any potential differential images/repos before going forward + if err := p.removeCopiesFromDifferentialPackage(); err != nil { + return err + } + } + + return nil +} + +func (p *Packager) assemble() error { + componentSBOMs := map[string]*layout.ComponentSBOM{} + var imageList []transform.Image + for idx, component := range p.cfg.Pkg.Components { + onCreate := component.Actions.OnCreate + onFailure := func() { + if err := p.runActions(onCreate.Defaults, onCreate.OnFailure, nil); err != nil { + message.Debugf("unable to run component failure action: %s", err.Error()) + } + } + if err := p.addComponent(idx, component); err != nil { + onFailure() + return fmt.Errorf("unable to add component %q: %w", component.Name, err) + } + + if err := p.runActions(onCreate.Defaults, onCreate.OnSuccess, nil); err != nil { + onFailure() + return fmt.Errorf("unable to run component success action: %w", err) + } + + if !p.cfg.CreateOpts.SkipSBOM { + componentSBOM, err := p.getFilesToSBOM(component) + if err != nil { + return fmt.Errorf("unable to create component SBOM: %w", err) + } + if componentSBOM != nil && len(componentSBOM.Files) > 0 { + componentSBOMs[component.Name] = componentSBOM + } + } + + // Combine all component images into a single entry for efficient layer reuse. + for _, src := range component.Images { + refInfo, err := transform.ParseImageRef(src) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", src, err) + } + imageList = append(imageList, refInfo) + } + } + + imageList = helpers.Unique(imageList) + var sbomImageList []transform.Image + + // Images are handled separately from other component assets. + if len(imageList) > 0 { + message.HeaderInfof("📦 PACKAGE IMAGES") + + p.layout = p.layout.AddImages() + + var pulled []images.ImgInfo + var err error + + doPull := func() error { + imgConfig := images.ImageConfig{ + ImagesPath: p.layout.Images.Base, + ImageList: imageList, + Insecure: config.CommonOptions.Insecure, + Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture}, + RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides, + } + + pulled, err = imgConfig.PullAll() + return err + } + + if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + } + + for _, imgInfo := range pulled { + if err := p.layout.Images.AddV1Image(imgInfo.Img); err != nil { + return err + } + if imgInfo.HasImageLayers { + sbomImageList = append(sbomImageList, imgInfo.RefInfo) + } + } + } + + // Ignore SBOM creation if the flag is set. + if p.cfg.CreateOpts.SkipSBOM { + message.Debug("Skipping image SBOM processing per --skip-sbom flag") + } else { + p.layout = p.layout.AddSBOMs() + if err := sbom.Catalog(componentSBOMs, sbomImageList, p.layout); err != nil { + return fmt.Errorf("unable to create an SBOM catalog for the package: %w", err) + } + } + + return nil +} + +func (p *Packager) assembleSkeleton() error { + if err := p.skeletonizeExtensions(); err != nil { + return err + } + for _, warning := range p.warnings { + message.Warn(warning) + } + for idx, component := range p.cfg.Pkg.Components { + if err := p.addComponent(idx, component); err != nil { + return err + } + + if err := p.layout.Components.Archive(component, false); err != nil { + return err + } + } + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + return p.writeYaml() +} + +// output assumes it is running from cwd, not the build directory +func (p *Packager) output() error { + // Process the component directories into compressed tarballs + // NOTE: This is purposefully being done after the SBOM cataloging + for _, component := range p.cfg.Pkg.Components { + // Make the component a tar archive + if err := p.layout.Components.Archive(component, true); err != nil { + return fmt.Errorf("unable to archive component: %s", err.Error()) + } + } + + // Calculate all the checksums + checksumChecksum, err := p.generatePackageChecksums() + if err != nil { + return fmt.Errorf("unable to generate checksums for the package: %w", err) + } + p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum + + // Save the transformed config. + if err := p.writeYaml(); err != nil { + return fmt.Errorf("unable to write zarf.yaml: %w", err) + } + + // Sign the config file if a key was provided + if p.cfg.CreateOpts.SigningKeyPath != "" { + if err := p.signPackage(p.cfg.CreateOpts.SigningKeyPath, p.cfg.CreateOpts.SigningKeyPassword); err != nil { + return err + } + } + + // Create a remote ref + client for the package (if output is OCI) + // then publish the package to the remote. + if helpers.IsOCIURL(p.cfg.CreateOpts.Output) { + ref, err := oci.ReferenceFromMetadata(p.cfg.CreateOpts.Output, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) + if err != nil { + return err + } + remote, err := oci.NewOrasRemote(ref) + if err != nil { + return err + } + + err = remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency) + if err != nil { + return fmt.Errorf("unable to publish package: %w", err) + } + message.HorizontalRule() + flags := "" + if config.CommonOptions.Insecure { + flags = "--insecure" + } + message.Title("To inspect/deploy/pull:", "") + message.ZarfCommand("package inspect %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package deploy %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + message.ZarfCommand("package pull %s %s", helpers.OCIURLPrefix+remote.Repo().Reference.String(), flags) + } else { + // Use the output path if the user specified it. + packageName := filepath.Join(p.cfg.CreateOpts.Output, p.GetPackageName()) + + // Try to remove the package if it already exists. + _ = os.Remove(packageName) + + // Create the package tarball. + if err := p.archivePackage(packageName); err != nil { + return fmt.Errorf("unable to archive package: %w", err) + } + } + + // Output the SBOM files into a directory if specified. + if p.cfg.CreateOpts.ViewSBOM || p.cfg.CreateOpts.SBOMOutputDir != "" { + outputSBOM := p.cfg.CreateOpts.SBOMOutputDir + var sbomDir string + if err := p.layout.SBOMs.Unarchive(); err != nil { + return fmt.Errorf("unable to unarchive SBOMs: %w", err) + } + sbomDir = p.layout.SBOMs.Path + + if outputSBOM != "" { + out, err := sbom.OutputSBOMFiles(sbomDir, outputSBOM, p.cfg.Pkg.Metadata.Name) + if err != nil { + return err + } + sbomDir = out + } + + if p.cfg.CreateOpts.ViewSBOM { + sbom.ViewSBOMFiles(sbomDir) + } + } + return nil +} + +func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*layout.ComponentSBOM, error) { + componentPaths, err := p.layout.Components.Create(component) + if err != nil { + return nil, err + } + // Create an struct to hold the SBOM information for this component. + componentSBOM := &layout.ComponentSBOM{ + Files: []string{}, + Component: componentPaths, + } + + appendSBOMFiles := func(path string) { + if utils.IsDir(path) { + files, _ := utils.RecursiveFileList(path, nil, false) + componentSBOM.Files = append(componentSBOM.Files, files...) + } else { + componentSBOM.Files = append(componentSBOM.Files, path) + } + } + + for filesIdx, file := range component.Files { + path := filepath.Join(componentPaths.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + appendSBOMFiles(path) + } + + for dataIdx, data := range component.DataInjections { + path := filepath.Join(componentPaths.DataInjections, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + + appendSBOMFiles(path) + } + + return componentSBOM, nil +} + +func (p *Packager) addComponent(index int, component types.ZarfComponent) error { + message.HeaderInfof("📦 %s COMPONENT", strings.ToUpper(component.Name)) + + isSkeleton := p.cfg.CreateOpts.IsSkeleton + + componentPaths, err := p.layout.Components.Create(component) + if err != nil { + return err + } + + if isSkeleton && component.DeprecatedCosignKeyPath != "" { + dst := filepath.Join(componentPaths.Base, "cosign.pub") + err := utils.CreatePathAndCopy(component.DeprecatedCosignKeyPath, dst) + if err != nil { + return err + } + p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" + } + + // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. + if isSkeleton { + component.Actions.OnCreate.Defaults.Dir = "" + resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for idx := range actions { + actions[idx].Dir = nil + } + return actions + } + component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + } + + onCreate := component.Actions.OnCreate + if !isSkeleton { + if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { + return fmt.Errorf("unable to run component before action: %w", err) + } + } + + // If any helm charts are defined, process them. + for chartIdx, chart := range component.Charts { + + helmCfg := helm.New(chart, componentPaths.Charts, componentPaths.Values) + + if isSkeleton { + if chart.LocalPath != "" { + rel := filepath.Join(layout.ChartsDir, fmt.Sprintf("%s-%d", chart.Name, chartIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + err := utils.CreatePathAndCopy(chart.LocalPath, dst) + if err != nil { + return err + } + + p.cfg.Pkg.Components[index].Charts[chartIdx].LocalPath = rel + } + + for valuesIdx, path := range chart.ValuesFiles { + if helpers.IsURL(path) { + continue + } + + rel := fmt.Sprintf("%s-%d", helm.StandardName(layout.ValuesDir, chart), valuesIdx) + p.cfg.Pkg.Components[index].Charts[chartIdx].ValuesFiles[valuesIdx] = rel + + if err := utils.CreatePathAndCopy(path, filepath.Join(componentPaths.Base, rel)); err != nil { + return fmt.Errorf("unable to copy chart values file %s: %w", path, err) + } + } + } else { + err := helmCfg.PackageChart(componentPaths.Charts) + if err != nil { + return err + } + } + } + + for filesIdx, file := range component.Files { + message.Debugf("Loading %#v", file) + + rel := filepath.Join(layout.FilesDir, strconv.Itoa(filesIdx), filepath.Base(file.Target)) + dst := filepath.Join(componentPaths.Base, rel) + destinationDir := filepath.Dir(dst) + + if helpers.IsURL(file.Source) { + if isSkeleton { + continue + } + + if file.ExtractPath != "" { + + // get the compressedFileName from the source + compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) + if err != nil { + return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) + } + + compressedFile := filepath.Join(componentPaths.Temp, compressedFileName) + + // If the file is an archive, download it to the componentPath.Temp + if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + + err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) + if err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) + } + + } else { + if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + } + + } else { + if file.ExtractPath != "" { + if err := archiver.Extract(file.Source, file.ExtractPath, destinationDir); err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } + } + + } + + if file.ExtractPath != "" { + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + if err := os.Rename(updatedExtractedFileOrDir, dst); err != nil { + return fmt.Errorf(lang.ErrWritingFile, dst, err) + } + } + } + + if isSkeleton { + // Change the source to the new relative source directory (any remote files will have been skipped above) + p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel + // Remove the extractPath from a skeleton since it will already extract it + p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" + } + + // Abort packaging on invalid shasum (if one is specified). + if file.Shasum != "" { + if err := utils.SHAsMatch(dst, file.Shasum); err != nil { + return err + } + } + + if file.Executable || utils.IsDir(dst) { + _ = os.Chmod(dst, 0700) + } else { + _ = os.Chmod(dst, 0600) + } + } + + if len(component.DataInjections) > 0 { + spinner := message.NewProgressSpinner("Loading data injections") + defer spinner.Stop() + + for dataIdx, data := range component.DataInjections { + spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector) + + rel := filepath.Join(layout.DataInjectionsDir, strconv.Itoa(dataIdx), filepath.Base(data.Target.Path)) + dst := filepath.Join(componentPaths.Base, rel) + + if helpers.IsURL(data.Source) { + if isSkeleton { + continue + } + if err := utils.DownloadToFile(data.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, data.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(data.Source, dst); err != nil { + return fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error()) + } + if isSkeleton { + p.cfg.Pkg.Components[index].DataInjections[dataIdx].Source = rel + } + } + } + spinner.Success() + } + + if len(component.Manifests) > 0 { + // Get the proper count of total manifests to add. + manifestCount := 0 + + for _, manifest := range component.Manifests { + manifestCount += len(manifest.Files) + manifestCount += len(manifest.Kustomizations) + } + + spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount) + defer spinner.Stop() + + // Iterate over all manifests. + for manifestIdx, manifest := range component.Manifests { + for fileIdx, path := range manifest.Files { + rel := filepath.Join(layout.ManifestsDir, fmt.Sprintf("%s-%d.yaml", manifest.Name, fileIdx)) + dst := filepath.Join(componentPaths.Base, rel) + + // Copy manifests without any processing. + spinner.Updatef("Copying manifest %s", path) + if helpers.IsURL(path) { + if isSkeleton { + continue + } + if err := utils.DownloadToFile(path, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, path, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(path, dst); err != nil { + return fmt.Errorf("unable to copy manifest %s: %w", path, err) + } + if isSkeleton { + p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files[fileIdx] = rel + } + } + } + + for kustomizeIdx, path := range manifest.Kustomizations { + // Generate manifests from kustomizations and place in the package. + spinner.Updatef("Building kustomization for %s", path) + + kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, kustomizeIdx) + rel := filepath.Join(layout.ManifestsDir, kname) + dst := filepath.Join(componentPaths.Base, rel) + + if err := kustomize.Build(path, dst, manifest.KustomizeAllowAnyDirectory); err != nil { + return fmt.Errorf("unable to build kustomization %s: %w", path, err) + } + if isSkeleton { + p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files = append(p.cfg.Pkg.Components[index].Manifests[manifestIdx].Files, rel) + } + } + if isSkeleton { + // remove kustomizations + p.cfg.Pkg.Components[index].Manifests[manifestIdx].Kustomizations = nil + } + } + spinner.Success() + } + + // Load all specified git repos. + if len(component.Repos) > 0 && !isSkeleton { + spinner := message.NewProgressSpinner("Loading %d git repos", len(component.Repos)) + defer spinner.Stop() + + for _, url := range component.Repos { + // Pull all the references if there is no `@` in the string. + gitCfg := git.NewWithSpinner(types.GitServerInfo{}, spinner) + if err := gitCfg.Pull(url, componentPaths.Repos, false); err != nil { + return fmt.Errorf("unable to pull git repo %s: %w", url, err) + } + } + spinner.Success() + } + + if !isSkeleton { + if err := p.runActions(onCreate.Defaults, onCreate.After, nil); err != nil { + return fmt.Errorf("unable to run component after action: %w", err) + } + } + + return nil +} + +// generateChecksum walks through all of the files starting at the base path and generates a checksum file. +// Each file within the basePath represents a layer within the Zarf package. +// generateChecksum returns a SHA256 checksum of the checksums.txt file. +func (p *Packager) generatePackageChecksums() (string, error) { + // Loop over the "loaded" files + var checksumsData = []string{} + for rel, abs := range p.layout.Files() { + if rel == layout.ZarfYAML || rel == layout.Checksums { + continue + } + + sum, err := utils.GetSHA256OfFile(abs) + if err != nil { + return "", err + } + checksumsData = append(checksumsData, fmt.Sprintf("%s %s", sum, rel)) + } + slices.Sort(checksumsData) + + // Create the checksums file + checksumsFilePath := p.layout.Checksums + if err := utils.WriteFile(checksumsFilePath, []byte(strings.Join(checksumsData, "\n")+"\n")); err != nil { + return "", err + } + + // Calculate the checksum of the checksum file + return utils.GetSHA256OfFile(checksumsFilePath) +} + +// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package +func (p *Packager) loadDifferentialData() error { + // Save the fact that this is a differential build into the build data of the package + p.cfg.Pkg.Build.Differential = true + + tmpDir, _ := utils.MakeTempDir(config.CommonOptions.TempDirectory) + defer os.RemoveAll(tmpDir) + + // Load the package spec of the package we're using as a 'reference' for the differential build + if helpers.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) { + remote, err := oci.NewOrasRemote(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) + if err != nil { + return err + } + pkg, err := remote.FetchZarfYAML() + if err != nil { + return err + } + err = utils.WriteYaml(filepath.Join(tmpDir, layout.ZarfYAML), pkg, 0600) + if err != nil { + return err + } + } else { + if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, layout.ZarfYAML, tmpDir); err != nil { + return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error()) + } + } + + var differentialZarfConfig types.ZarfPackage + if err := utils.ReadYaml(filepath.Join(tmpDir, layout.ZarfYAML), &differentialZarfConfig); err != nil { + return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error()) + } + + // Generate a map of all the images and repos that are included in the provided package + allIncludedImagesMap := map[string]bool{} + allIncludedReposMap := map[string]bool{} + for _, component := range differentialZarfConfig.Components { + for _, image := range component.Images { + allIncludedImagesMap[image] = true + } + for _, repo := range component.Repos { + allIncludedReposMap[repo] = true + } + } + + p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap + p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap + p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version + + return nil +} + +// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package +func (p *Packager) removeCopiesFromDifferentialPackage() error { + // If a differential build was not requested, continue on as normal + if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" { + return nil + } + + // Loop through all of the components to determine if any of them are using already included images or repos + componentMap := make(map[int]types.ZarfComponent) + for idx, component := range p.cfg.Pkg.Components { + newImageList := []string{} + newRepoList := []string{} + // Generate a list of all unique images for this component + for _, img := range component.Images { + // If a image doesn't have a ref (or is a commonly reused ref), we will include this image in the differential package + imgRef, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error()) + } + + // Only include new images or images that have a commonly overwritten tag + imgTag := imgRef.TagOrDigest + useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly" + if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] { + newImageList = append(newImageList, img) + } else { + message.Debugf("Image %s is already included in the differential package", img) + } + } + + // Generate a list of all unique repos for this component + for _, repoURL := range component.Repos { + // Split the remote url and the zarf reference + _, refPlain, err := transform.GitURLSplitRef(repoURL) + if err != nil { + return err + } + + var ref plumbing.ReferenceName + // Parse the ref from the git URL. + if refPlain != "" { + ref = git.ParseRef(refPlain) + } + + // Only include new repos or repos that were not referenced by a specific commit sha or tag + useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain)) + if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] { + newRepoList = append(newRepoList, repoURL) + } else { + message.Debugf("Repo %s is already included in the differential package", repoURL) + } + } + + // Update the component with the unique lists of repos and images + component.Images = newImageList + component.Repos = newRepoList + componentMap[idx] = component + } + + // Update the package with the new component list + for idx, component := range componentMap { + p.cfg.Pkg.Components[idx] = component + } + + return nil +} diff --git a/src/pkg/packager/deploy.go b/src/pkg/packager/deploy.go index 8835e0f92e..f76d20962a 100644 --- a/src/pkg/packager/deploy.go +++ b/src/pkg/packager/deploy.go @@ -30,6 +30,14 @@ import ( corev1 "k8s.io/api/core/v1" ) +func (p *Packager) resetRegistryHPA() { + if p.isConnectedToCluster() && p.hpaModified { + if err := p.cluster.EnableRegHPAScaleDown(); err != nil { + message.Debugf("unable to reenable the registry HPA scale down: %s", err.Error()) + } + } +} + // Deploy attempts to deploy the given PackageConfig. func (p *Packager) Deploy() (err error) { if err = p.source.LoadPackage(p.layout, true); err != nil { @@ -61,13 +69,7 @@ func (p *Packager) Deploy() (err error) { p.hpaModified = false p.connectStrings = make(types.ConnectStrings) // Reset registry HPA scale down whether an error occurs or not - defer func() { - if p.isConnectedToCluster() && p.hpaModified { - if err := p.cluster.EnableRegHPAScaleDown(); err != nil { - message.Debugf("unable to reenable the registry HPA scale down: %s", err.Error()) - } - } - }() + defer p.resetRegistryHPA() // Filter out components that are not compatible with this system p.filterComponents() @@ -91,7 +93,7 @@ func (p *Packager) Deploy() (err error) { // deployComponents loops through a list of ZarfComponents and deploys them. func (p *Packager) deployComponents() (deployedComponents []types.DeployedComponent, err error) { - componentsToDeploy := p.getValidComponents() + componentsToDeploy := p.getSelectedComponents() // Generate a value template if p.valueTemplate, err = template.Generate(p.cfg); err != nil { @@ -113,7 +115,7 @@ func (p *Packager) deployComponents() (deployedComponents []types.DeployedCompon } // If this component requires a cluster, connect to one - if p.requiresCluster(component) { + if requiresCluster(component) { timeout := cluster.DefaultTimeout if p.isInitConfig() { timeout = 5 * time.Minute @@ -199,7 +201,7 @@ func (p *Packager) deployInitComponent(component types.ZarfComponent) (charts [] isAgent := component.Name == "zarf-agent" // Always init the state before the first component that requires the cluster (on most deployments, the zarf-seed-registry) - if p.requiresCluster(component) && p.cfg.State == nil { + if requiresCluster(component) && p.cfg.State == nil { err = p.cluster.InitZarfState(p.cfg.InitOpts) if err != nil { return charts, fmt.Errorf("unable to initialize Zarf state: %w", err) @@ -212,7 +214,7 @@ func (p *Packager) deployInitComponent(component types.ZarfComponent) (charts [] } if isRegistry { - // If we are deploying the registry then mark the HPA as "modifed" to set it to Min later + // If we are deploying the registry then mark the HPA as "modified" to set it to Min later p.hpaModified = true } @@ -263,7 +265,7 @@ func (p *Packager) deployComponent(component types.ZarfComponent, noImgChecksum } } - if !p.valueTemplate.Ready() && p.requiresCluster(component) { + if !p.valueTemplate.Ready() && requiresCluster(component) { // Setup the state in the config and get the valuesTemplate p.valueTemplate, err = p.setupStateValuesTemplate() if err != nil { @@ -474,6 +476,9 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error gitClient := git.New(p.cfg.State.GitServer) svcInfo, _ := k8s.ServiceInfoFromServiceURL(gitClient.Server.Address) + var err error + var tunnel *k8s.Tunnel + // If this is a service (svcInfo is not nil), create a port-forward tunnel to that resource if svcInfo != nil { if !p.isConnectedToCluster() { @@ -483,7 +488,7 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error } } - tunnel, err := p.cluster.NewTunnel(svcInfo.Namespace, k8s.SvcResource, svcInfo.Name, "", 0, svcInfo.Port) + tunnel, err = p.cluster.NewTunnel(svcInfo.Namespace, k8s.SvcResource, svcInfo.Name, "", 0, svcInfo.Port) if err != nil { return err } @@ -494,6 +499,8 @@ func (p *Packager) pushReposToRepository(reposPath string, repos []string) error } defer tunnel.Close() gitClient.Server.Address = tunnel.HTTPEndpoint() + + return tunnel.Wrap(func() error { return gitClient.PushRepo(repoURL, reposPath) }) } return gitClient.PushRepo(repoURL, reposPath) diff --git a/src/pkg/packager/dev.go b/src/pkg/packager/dev.go new file mode 100644 index 0000000000..b4e1744d46 --- /dev/null +++ b/src/pkg/packager/dev.go @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager contains functions for interacting with, managing and deploying Zarf packages. +package packager + +import ( + "fmt" + "os" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// DevDeploy creates + deploys a package in one shot +func (p *Packager) DevDeploy() error { + config.CommonOptions.Confirm = true + p.cfg.CreateOpts.SkipSBOM = !p.cfg.CreateOpts.NoYOLO + + cwd, err := os.Getwd() + if err != nil { + return err + } + + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + return err + } + + if err := p.load(); err != nil { + return err + } + + // Filter out components that are not compatible with this system + p.filterComponents() + + // Also filter out components that are not required, nor requested via --components + // This is different from the above filter, as it is not based on the system, but rather + // the user's selection and the component's `required` field + // This is also different from regular package creation, where we still assemble and package up + // all components and their dependencies, regardless of whether they are required or not + p.cfg.Pkg.Components = p.getSelectedComponents() + + if err := validate.Run(p.cfg.Pkg); err != nil { + return fmt.Errorf("unable to validate package: %w", err) + } + + // If building in yolo mode, strip out all images and repos + if !p.cfg.CreateOpts.NoYOLO { + for idx := range p.cfg.Pkg.Components { + p.cfg.Pkg.Components[idx].Images = []string{} + p.cfg.Pkg.Components[idx].Repos = []string{} + } + } + + if err := p.assemble(); err != nil { + return err + } + + message.HeaderInfof("📦 PACKAGE DEPLOY %s", p.cfg.Pkg.Metadata.Name) + + // Set variables and prompt if --confirm is not set + if err := p.setVariableMapInConfig(); err != nil { + return fmt.Errorf("unable to set the active variables: %w", err) + } + + p.connectStrings = make(types.ConnectStrings) + + if !p.cfg.CreateOpts.NoYOLO { + p.cfg.Pkg.Metadata.YOLO = true + } else { + p.hpaModified = false + // Reset registry HPA scale down whether an error occurs or not + defer p.resetRegistryHPA() + } + + // Get a list of all the components we are deploying and actually deploy them + deployedComponents, err := p.deployComponents() + if err != nil { + return err + } + if len(deployedComponents) == 0 { + message.Warn("No components were selected for deployment. Inspect the package to view the available components and select components interactively or by name with \"--components\"") + } + + // Notify all the things about the successful deployment + message.Successf("Zarf dev deployment complete") + + message.HorizontalRule() + message.Title("Next steps:", "") + + message.ZarfCommand("package inspect %s", p.cfg.Pkg.Metadata.Name) + + // cd back + return os.Chdir(cwd) +} diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index b9142cc3fb..bc8b5df336 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -7,12 +7,19 @@ package lint import ( "embed" "fmt" + "os" "path/filepath" "regexp" "strings" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/packager" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" + "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/xeipuuv/gojsonschema" ) @@ -24,22 +31,29 @@ func getSchemaFile() ([]byte, error) { return ZarfSchema.ReadFile("zarf.schema.json") } -// ValidateZarfSchema validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist +// Validate validates a zarf file against the zarf schema, returns *validator with warnings or errors if they exist // along with an error if the validation itself failed -func ValidateZarfSchema(path string) (*Validator, error) { +func Validate(createOpts types.ZarfCreateOptions) (*Validator, error) { validator := Validator{} var err error - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { + + if err := utils.ReadYaml(filepath.Join(createOpts.BaseDir, layout.ZarfYAML), &validator.typedZarfPackage); err != nil { return nil, err } - checkForVarInComponentImport(&validator) - - if validator.jsonSchema, err = getSchemaFile(); err != nil { + if err := utils.ReadYaml(filepath.Join(createOpts.BaseDir, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { return nil, err } - if err := utils.ReadYaml(filepath.Join(path, layout.ZarfYAML), &validator.untypedZarfPackage); err != nil { + if err := os.Chdir(createOpts.BaseDir); err != nil { + return nil, fmt.Errorf("unable to access directory '%s': %w", createOpts.BaseDir, err) + } + + validator.baseDir = createOpts.BaseDir + + lintComponents(&validator, &createOpts) + + if validator.jsonSchema, err = getSchemaFile(); err != nil { return nil, err } @@ -50,16 +64,194 @@ func ValidateZarfSchema(path string) (*Validator, error) { return &validator, nil } -func checkForVarInComponentImport(validator *Validator) { +func lintComponents(validator *Validator, createOpts *types.ZarfCreateOptions) { for i, component := range validator.typedZarfPackage.Components { - if strings.Contains(component.Import.Path, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.path: Will not resolve ZARF_PKG_TMPL_* variables", i)) + arch := config.GetArch(validator.typedZarfPackage.Metadata.Architecture) + + if !composer.CompatibleComponent(component, arch, createOpts.Flavor) { + continue } - if strings.Contains(component.Import.URL, types.ZarfPackageTemplatePrefix) { - validator.addWarning(fmt.Sprintf(".components.[%d].import.url: Will not resolve ZARF_PKG_TMPL_* variables", i)) + + chain, err := composer.NewImportChain(component, i, validator.typedZarfPackage.Metadata.Name, arch, createOpts.Flavor) + baseComponent := chain.Head() + + var badImportYqPath string + if baseComponent != nil { + if baseComponent.Import.URL != "" { + badImportYqPath = fmt.Sprintf(".components.[%d].import.url", i) + } + if baseComponent.Import.Path != "" { + badImportYqPath = fmt.Sprintf(".components.[%d].import.path", i) + } + } + if err != nil { + validator.addError(validatorMessage{ + description: err.Error(), + packageRelPath: ".", + packageName: validator.typedZarfPackage.Metadata.Name, + yqPath: badImportYqPath, + }) + } + + node := baseComponent + for node != nil { + checkForVarInComponentImport(validator, node) + fillComponentTemplate(validator, node, createOpts) + lintComponent(validator, node) + node = node.Next() } } +} + +func fillComponentTemplate(validator *Validator, node *composer.Node, createOpts *types.ZarfCreateOptions) { + + err := packager.ReloadComponentTemplate(&node.ZarfComponent) + if err != nil { + validator.addWarning(validatorMessage{ + description: err.Error(), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + templateMap := map[string]string{} + + setVarsAndWarn := func(templatePrefix string, deprecated bool) { + yamlTemplates, err := utils.FindYamlTemplates(node, templatePrefix, "###") + if err != nil { + validator.addWarning(validatorMessage{ + description: err.Error(), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + + for key := range yamlTemplates { + if deprecated { + validator.addWarning(validatorMessage{ + description: fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + _, present := createOpts.SetVariables[key] + if !present { + validator.addWarning(validatorMessage{ + description: lang.UnsetVarLintWarning, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + }) + } + } + for key, value := range createOpts.SetVariables { + templateMap[fmt.Sprintf("%s%s###", templatePrefix, key)] = value + } + } + + setVarsAndWarn(types.ZarfPackageTemplatePrefix, false) + + // [DEPRECATION] Set the Package Variable syntax as well for backward compatibility + setVarsAndWarn(types.ZarfPackageVariablePrefix, true) + utils.ReloadYamlTemplate(node, templateMap) +} + +func isPinnedImage(image string) (bool, error) { + transformedImage, err := transform.ParseImageRef(image) + if err != nil { + if strings.Contains(image, types.ZarfPackageTemplatePrefix) || + strings.Contains(image, types.ZarfPackageVariablePrefix) { + return true, nil + } + return false, err + } + return (transformedImage.Digest != ""), err +} + +func isPinnedRepo(repo string) bool { + return (strings.Contains(repo, "@")) +} + +func lintComponent(validator *Validator, node *composer.Node) { + checkForUnpinnedRepos(validator, node) + checkForUnpinnedImages(validator, node) + checkForUnpinnedFiles(validator, node) +} + +func checkForUnpinnedRepos(validator *Validator, node *composer.Node) { + for j, repo := range node.Repos { + repoYqPath := fmt.Sprintf(".components.[%d].repos.[%d]", node.GetIndex(), j) + if !isPinnedRepo(repo) { + validator.addWarning(validatorMessage{ + yqPath: repoYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Unpinned repository", + item: repo, + }) + } + } +} + +func checkForUnpinnedImages(validator *Validator, node *composer.Node) { + for j, image := range node.Images { + imageYqPath := fmt.Sprintf(".components.[%d].images.[%d]", node.GetIndex(), j) + pinnedImage, err := isPinnedImage(image) + if err != nil { + validator.addError(validatorMessage{ + yqPath: imageYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Invalid image reference", + item: image, + }) + continue + } + if !pinnedImage { + validator.addWarning(validatorMessage{ + yqPath: imageYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Image not pinned with digest", + item: image, + }) + } + } +} + +func checkForUnpinnedFiles(validator *Validator, node *composer.Node) { + for j, file := range node.Files { + fileYqPath := fmt.Sprintf(".components.[%d].files.[%d]", node.GetIndex(), j) + if file.Shasum == "" && helpers.IsURL(file.Source) { + validator.addWarning(validatorMessage{ + yqPath: fileYqPath, + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "No shasum for remote file", + item: file.Source, + }) + } + } +} + +func checkForVarInComponentImport(validator *Validator, node *composer.Node) { + if strings.Contains(node.Import.Path, types.ZarfPackageTemplatePrefix) { + validator.addWarning(validatorMessage{ + yqPath: fmt.Sprintf(".components.[%d].import.path", node.GetIndex()), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Zarf does not evaluate variables at component.x.import.path", + item: node.Import.Path, + }) + } + if strings.Contains(node.Import.URL, types.ZarfPackageTemplatePrefix) { + validator.addWarning(validatorMessage{ + yqPath: fmt.Sprintf(".components.[%d].import.url", node.GetIndex()), + packageRelPath: node.ImportLocation(), + packageName: node.GetOriginalPackageName(), + description: "Zarf does not evaluate variables at component.x.import.url", + item: node.Import.URL, + }) + } } func makeFieldPathYqCompat(field string) string { @@ -86,9 +278,12 @@ func validateSchema(validator *Validator) error { if !result.Valid() { for _, desc := range result.Errors() { - err := fmt.Errorf( - "%s: %s", makeFieldPathYqCompat(desc.Field()), desc.Description()) - validator.addError(err) + validator.addError(validatorMessage{ + yqPath: makeFieldPathYqCompat(desc.Field()), + description: desc.Description(), + packageRelPath: ".", + packageName: validator.typedZarfPackage.Metadata.Name, + }) } } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index ddc9907e0f..5f81660d32 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -5,9 +5,14 @@ package lint import ( + "errors" + "fmt" "os" + "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" "github.com/stretchr/testify/require" @@ -26,14 +31,6 @@ components: - name: import-test import: path: 123123 - -- name: import-test - import: - path: "###ZARF_PKG_TMPL_ZEBRA###" - -- name: import-url - import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" ` const goodZarfPackage = ` @@ -71,7 +68,7 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.Empty(t, validator.errors) + require.Empty(t, validator.findings) }) t.Run("validate schema fail", func(t *testing.T) { @@ -79,23 +76,82 @@ func TestValidateSchema(t *testing.T) { validator := Validator{untypedZarfPackage: unmarshalledYaml, jsonSchema: getZarfSchema(t)} err := validateSchema(&validator) require.NoError(t, err) - require.EqualError(t, validator.errors[0], ".components.[0].import: Additional property not-path is not allowed") - require.EqualError(t, validator.errors[1], ".components.[1].import.path: Invalid type. Expected: string, given: integer") + config.NoColor = true + require.Equal(t, "Additional property not-path is not allowed", validator.findings[0].String()) + require.Equal(t, "Invalid type. Expected: string, given: integer", validator.findings[1].String()) }) t.Run("Template in component import success", func(t *testing.T) { unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, goodZarfPackage) validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForVarInComponentImport(&validator) - require.Empty(t, validator.warnings) + for _, component := range validator.typedZarfPackage.Components { + lintComponent(&validator, &composer.Node{ZarfComponent: component}) + } + require.Empty(t, validator.findings) }) - t.Run("Template in component import failure", func(t *testing.T) { - unmarshalledYaml := readAndUnmarshalYaml[types.ZarfPackage](t, badZarfPackage) - validator := Validator{typedZarfPackage: unmarshalledYaml} - checkForVarInComponentImport(&validator) - require.Equal(t, validator.warnings[0], ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Equal(t, validator.warnings[1], ".components.[3].import.url: Will not resolve ZARF_PKG_TMPL_* variables") + t.Run("Path template in component import failure", func(t *testing.T) { + pathVar := "###ZARF_PKG_TMPL_PATH###" + pathComponent := types.ZarfComponent{Import: types.ZarfComponentImport{Path: pathVar}} + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}}} + checkForVarInComponentImport(&validator, &composer.Node{ZarfComponent: pathComponent}) + require.Equal(t, pathVar, validator.findings[0].item) + }) + + t.Run("OCI template in component import failure", func(t *testing.T) { + ociPathVar := "oci://###ZARF_PKG_TMPL_PATH###" + URLComponent := types.ZarfComponent{Import: types.ZarfComponentImport{URL: ociPathVar}} + validator := Validator{typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{URLComponent}}} + checkForVarInComponentImport(&validator, &composer.Node{ZarfComponent: URLComponent}) + require.Equal(t, ociPathVar, validator.findings[0].item) + }) + + t.Run("Unpinnned repo warning", func(t *testing.T) { + validator := Validator{} + unpinnedRepo := "https://github.com/defenseunicorns/zarf-public-test.git" + component := types.ZarfComponent{Repos: []string{ + unpinnedRepo, + "https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1"}} + checkForUnpinnedRepos(&validator, &composer.Node{ZarfComponent: component}) + require.Equal(t, unpinnedRepo, validator.findings[0].item) + require.Equal(t, len(validator.findings), 1) + }) + + t.Run("Unpinnned image warning", func(t *testing.T) { + validator := Validator{} + unpinnedImage := "registry.com:9001/whatever/image:1.0.0" + badImage := "badimage:badimage@@sha256:3fbc632167424a6d997e74f5" + component := types.ZarfComponent{Images: []string{ + unpinnedImage, + "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", + badImage}} + checkForUnpinnedImages(&validator, &composer.Node{ZarfComponent: component}) + require.Equal(t, unpinnedImage, validator.findings[0].item) + require.Equal(t, badImage, validator.findings[1].item) + require.Equal(t, 2, len(validator.findings)) + + }) + + t.Run("Unpinnned file warning", func(t *testing.T) { + validator := Validator{} + fileURL := "http://example.com/file.zip" + localFile := "local.txt" + zarfFiles := []types.ZarfFile{ + { + Source: fileURL, + }, + { + Source: localFile, + }, + { + Source: fileURL, + Shasum: "fake-shasum", + }, + } + component := types.ZarfComponent{Files: zarfFiles} + checkForUnpinnedFiles(&validator, &composer.Node{ZarfComponent: component}) + require.Equal(t, fileURL, validator.findings[0].item) + require.Equal(t, 1, len(validator.findings)) }) t.Run("Wrap standalone numbers in bracket", func(t *testing.T) { @@ -110,4 +166,68 @@ func TestValidateSchema(t *testing.T) { acutal := makeFieldPathYqCompat(input) require.Equal(t, input, acutal) }) + + t.Run("Test composable components", func(t *testing.T) { + pathVar := "fake-path" + unpinnedImage := "unpinned:latest" + pathComponent := types.ZarfComponent{ + Import: types.ZarfComponentImport{Path: pathVar}, + Images: []string{unpinnedImage}} + validator := Validator{ + typedZarfPackage: types.ZarfPackage{Components: []types.ZarfComponent{pathComponent}, + Metadata: types.ZarfMetadata{Name: "test-zarf-package"}}} + + createOpts := types.ZarfCreateOptions{Flavor: "", BaseDir: "."} + lintComponents(&validator, &createOpts) + // Require.contains rather than equals since the error message changes from linux to windows + require.Contains(t, validator.findings[0].description, fmt.Sprintf("open %s", filepath.Join("fake-path", "zarf.yaml"))) + require.Equal(t, ".components.[0].import.path", validator.findings[0].yqPath) + require.Equal(t, ".", validator.findings[0].packageRelPath) + require.Equal(t, unpinnedImage, validator.findings[1].item) + require.Equal(t, ".", validator.findings[1].packageRelPath) + }) + + t.Run("isImagePinned", func(t *testing.T) { + t.Parallel() + tests := []struct { + input string + expected bool + err error + }{ + { + input: "registry.com:8080/defenseunicorns/whatever", + expected: false, + err: nil, + }, + { + input: "ghcr.io/defenseunicorns/pepr/controller:v0.15.0", + expected: false, + err: nil, + }, + { + input: "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", + expected: true, + err: nil, + }, + { + input: "busybox:bad/image", + expected: false, + err: errors.New("invalid reference format"), + }, + { + input: "busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE###", + expected: true, + err: nil, + }, + } + for _, tc := range tests { + t.Run(tc.input, func(t *testing.T) { + acutal, err := isPinnedImage(tc.input) + if err != nil { + require.EqualError(t, err, tc.err.Error()) + } + require.Equal(t, tc.expected, acutal) + }) + } + }) } diff --git a/src/pkg/packager/lint/validator.go b/src/pkg/packager/lint/validator.go index f88c22de5f..830ac21ff2 100644 --- a/src/pkg/packager/lint/validator.go +++ b/src/pkg/packager/lint/validator.go @@ -6,63 +6,146 @@ package lint import ( "fmt" + "path/filepath" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/fatih/color" ) +type category int + +const ( + categoryError category = 1 + categoryWarning category = 2 +) + +type validatorMessage struct { + yqPath string + description string + item string + packageRelPath string + packageName string + category category +} + +func (c category) String() string { + if c == categoryError { + return message.ColorWrap("Error", color.FgRed) + } else if c == categoryWarning { + return message.ColorWrap("Warning", color.FgYellow) + } + return "" +} + +func (vm validatorMessage) String() string { + if vm.item != "" { + vm.item = fmt.Sprintf(" - %s", vm.item) + } + return fmt.Sprintf("%s%s", vm.description, vm.item) +} + // Validator holds the warnings/errors and messaging that we get from validation type Validator struct { - warnings []string - errors []error + findings []validatorMessage jsonSchema []byte typedZarfPackage types.ZarfPackage untypedZarfPackage interface{} + baseDir string } // DisplayFormattedMessage message sent to user based on validator results func (v Validator) DisplayFormattedMessage() { - if !v.hasWarnings() && !v.hasErrors() { - message.Successf("Schema validation successful for %q", v.typedZarfPackage.Metadata.Name) + if !v.hasFindings() { + message.Successf("0 findings for %q", v.typedZarfPackage.Metadata.Name) } v.printValidationTable() } // IsSuccess returns true if there are not any errors func (v Validator) IsSuccess() bool { - return !v.hasErrors() + for _, finding := range v.findings { + if finding.category == categoryError { + return false + } + } + return true +} + +func (v Validator) packageRelPathToUser(vm validatorMessage) string { + if helpers.IsOCIURL(vm.packageRelPath) { + return vm.packageRelPath + } + return filepath.Join(v.baseDir, vm.packageRelPath) } func (v Validator) printValidationTable() { - if v.hasWarnings() || v.hasErrors() { - header := []string{"Type", "Message"} - connectData := [][]string{} - for _, warning := range v.warnings { - connectData = append(connectData, []string{utils.ColorWrap("Warning", color.FgYellow), warning}) + if !v.hasFindings() { + return + } + + mapOfFindingsByPath := make(map[string][]validatorMessage) + for _, finding := range v.findings { + mapOfFindingsByPath[finding.packageRelPath] = append(mapOfFindingsByPath[finding.packageRelPath], finding) + } + + header := []string{"Type", "Path", "Message"} + + for packageRelPath, findings := range mapOfFindingsByPath { + lintData := [][]string{} + for _, finding := range findings { + lintData = append(lintData, []string{finding.category.String(), finding.getPath(), finding.String()}) + } + message.Notef("Linting package %q at %s", findings[0].packageName, v.packageRelPathToUser(findings[0])) + message.Table(header, lintData) + message.Info(v.getFormattedFindingCount(packageRelPath, findings[0].packageName)) + } +} + +func (v Validator) getFormattedFindingCount(relPath string, packageName string) string { + warningCount := 0 + errorCount := 0 + for _, finding := range v.findings { + if finding.packageRelPath != relPath { + continue + } + if finding.category == categoryWarning { + warningCount++ } - for _, err := range v.errors { - connectData = append(connectData, []string{utils.ColorWrap("Error", color.FgRed), err.Error()}) + if finding.category == categoryError { + errorCount++ } - message.Table(header, connectData) - message.Info(fmt.Sprintf("%d warnings and %d errors in %q", - len(v.warnings), len(v.errors), v.typedZarfPackage.Metadata.Name)) } + wordWarning := "warnings" + if warningCount == 1 { + wordWarning = "warning" + } + wordError := "errors" + if errorCount == 1 { + wordError = "error" + } + return fmt.Sprintf("%d %s and %d %s in %q", + warningCount, wordWarning, errorCount, wordError, packageName) } -func (v Validator) hasWarnings() bool { - return len(v.warnings) > 0 +func (vm validatorMessage) getPath() string { + if vm.yqPath == "" { + return "" + } + return message.ColorWrap(vm.yqPath, color.FgCyan) } -func (v Validator) hasErrors() bool { - return len(v.errors) > 0 +func (v Validator) hasFindings() bool { + return len(v.findings) > 0 } -func (v *Validator) addWarning(message string) { - v.warnings = append(v.warnings, message) +func (v *Validator) addWarning(vmessage validatorMessage) { + vmessage.category = categoryWarning + v.findings = helpers.Unique(append(v.findings, vmessage)) } -func (v *Validator) addError(err error) { - v.errors = append(v.errors, err) +func (v *Validator) addError(vMessage validatorMessage) { + vMessage.category = categoryError + v.findings = helpers.Unique(append(v.findings, vMessage)) } diff --git a/src/pkg/packager/mirror.go b/src/pkg/packager/mirror.go index 293663a97c..689ddcfbaa 100644 --- a/src/pkg/packager/mirror.go +++ b/src/pkg/packager/mirror.go @@ -8,11 +8,8 @@ import ( "fmt" "strings" - "slices" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -45,17 +42,9 @@ func (p *Packager) Mirror() (err error) { // Filter out components that are not compatible with this system if we have loaded from a tarball p.filterComponents() - requestedComponentNames := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) - for _, component := range p.cfg.Pkg.Components { - if len(requestedComponentNames) == 0 || slices.Contains(requestedComponentNames, component.Name) { - if err := p.mirrorComponent(component); err != nil { - return err - } - } - } - - return nil + // Run mirror for each requested component + return p.forIncludedComponents(p.mirrorComponent) } // mirrorComponent mirrors a Zarf Component. diff --git a/src/pkg/packager/publish.go b/src/pkg/packager/publish.go index 2eacd46f6b..6022eeada2 100644 --- a/src/pkg/packager/publish.go +++ b/src/pkg/packager/publish.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" @@ -31,20 +30,34 @@ func (p *Packager) Publish() (err error) { // oci --> oci is a special case, where we will use oci.CopyPackage so that we can transfer the package // w/o layers touching the filesystem srcRemote := p.source.(*sources.OCISource).OrasRemote - srcRemote.WithContext(ctx) parts := strings.Split(srcRemote.Repo().Reference.Repository, "/") packageName := parts[len(parts)-1] p.cfg.PublishOpts.PackageDestination = p.cfg.PublishOpts.PackageDestination + "/" + packageName - err = p.setOCIRemote(p.cfg.PublishOpts.PackageDestination) + arch := config.GetArch() + dstRemote, err := oci.NewOrasRemote(p.cfg.PublishOpts.PackageDestination, oci.WithArch(arch)) if err != nil { return err } - p.remote.WithContext(ctx) - if err := oci.CopyPackage(ctx, srcRemote, p.remote, nil, config.CommonOptions.OCIConcurrency); err != nil { + srcRoot, err := srcRemote.ResolveRoot() + if err != nil { + return err + } + + pkg, err := srcRemote.FetchZarfYAML() + if err != nil { + return err + } + + // ensure cli arch matches package arch + if pkg.Build.Architecture != arch { + return fmt.Errorf("architecture mismatch (specified: %q, found %q)", arch, pkg.Build.Architecture) + } + + if err := oci.CopyPackage(ctx, srcRemote, dstRemote, nil, config.CommonOptions.OCIConcurrency); err != nil { return err } @@ -58,21 +71,32 @@ func (p *Packager) Publish() (err error) { } expected := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) - // tag the manifest the same as the source - if err := p.remote.Repo().Manifests().PushReference(ctx, expected, bytes.NewReader(b), srcRemote.Repo().Reference.Reference); err != nil { + if err := dstRemote.Repo().Manifests().PushReference(ctx, expected, bytes.NewReader(b), srcRoot.Digest.String()); err != nil { + return err + } + + tag := srcRemote.Repo().Reference.Reference + if err := dstRemote.UpdateIndex(tag, arch, expected); err != nil { return err } - message.Infof("Published %s to %s", srcRemote.Repo().Reference, p.remote.Repo().Reference) + message.Infof("Published %s to %s", srcRemote.Repo().Reference, dstRemote.Repo().Reference) return nil } - var referenceSuffix string - if p.cfg.CreateOpts.BaseDir != "" { - referenceSuffix = oci.SkeletonSuffix - err := p.loadSkeleton() + if p.cfg.CreateOpts.IsSkeleton { + cwd, err := os.Getwd() if err != nil { return err } + if err := p.cdToBaseDir(p.cfg.CreateOpts.BaseDir, cwd); err != nil { + return err + } + if err := p.load(); err != nil { + return err + } + if err := p.assembleSkeleton(); err != nil { + return err + } } else { if err = p.source.LoadPackage(p.layout, false); err != nil { return fmt.Errorf("unable to load the package: %w", err) @@ -80,17 +104,15 @@ func (p *Packager) Publish() (err error) { if err = p.readZarfYAML(p.layout.ZarfYAML); err != nil { return err } - - referenceSuffix = p.arch } // Get a reference to the registry for this package - ref, err := oci.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, referenceSuffix) + ref, err := oci.ReferenceFromMetadata(p.cfg.PublishOpts.PackageDestination, &p.cfg.Pkg.Metadata, &p.cfg.Pkg.Build) if err != nil { return err } - err = p.setOCIRemote(ref) + remote, err := oci.NewOrasRemote(ref) if err != nil { return err } @@ -105,10 +127,10 @@ func (p *Packager) Publish() (err error) { message.HeaderInfof("📦 PACKAGE PUBLISH %s:%s", p.cfg.Pkg.Metadata.Name, ref) // Publish the package/skeleton to the registry - if err := p.remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency); err != nil { + if err := remote.PublishPackage(&p.cfg.Pkg, p.layout, config.CommonOptions.OCIConcurrency); err != nil { return err } - if strings.HasSuffix(p.remote.Repo().Reference.String(), oci.SkeletonSuffix) { + if p.cfg.CreateOpts.IsSkeleton { message.Title("How to import components from this skeleton:", "") ex := []types.ZarfComponent{} for _, c := range p.cfg.Pkg.Components { @@ -116,7 +138,7 @@ func (p *Packager) Publish() (err error) { Name: fmt.Sprintf("import-%s", c.Name), Import: types.ZarfComponentImport{ ComponentName: c.Name, - URL: helpers.OCIURLPrefix + p.remote.Repo().Reference.String(), + URL: helpers.OCIURLPrefix + remote.Repo().Reference.String(), }, }) } @@ -124,49 +146,3 @@ func (p *Packager) Publish() (err error) { } return nil } - -func (p *Packager) loadSkeleton() (err error) { - if err := os.Chdir(p.cfg.CreateOpts.BaseDir); err != nil { - return err - } - if err = p.readZarfYAML(layout.ZarfYAML); err != nil { - return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error()) - } - - if p.isInitConfig() { - p.cfg.Pkg.Metadata.Version = config.CLIVersion - } - - err = p.composeComponents() - if err != nil { - return err - } - - err = p.skeletonizeExtensions() - if err != nil { - return err - } - - for _, warning := range p.warnings { - message.Warn(warning) - } - - for idx, component := range p.cfg.Pkg.Components { - isSkeleton := true - if err := p.addComponent(idx, component, isSkeleton); err != nil { - return err - } - - if err := p.layout.Components.Archive(component, false); err != nil { - return err - } - } - - checksumChecksum, err := p.generatePackageChecksums() - if err != nil { - return fmt.Errorf("unable to generate checksums for skeleton package: %w", err) - } - p.cfg.Pkg.Metadata.AggregateChecksum = checksumChecksum - - return p.writeYaml() -} diff --git a/src/pkg/packager/remove.go b/src/pkg/packager/remove.go index 89b851eaf0..2aaee0c3af 100644 --- a/src/pkg/packager/remove.go +++ b/src/pkg/packager/remove.go @@ -24,17 +24,13 @@ import ( // Remove removes a package that was already deployed onto a cluster, uninstalling all installed helm charts. func (p *Packager) Remove() (err error) { - _, requiresCluster := p.source.(*sources.ClusterSource) - if requiresCluster { + _, isClusterSource := p.source.(*sources.ClusterSource) + if isClusterSource { p.cluster = p.source.(*sources.ClusterSource).Cluster } spinner := message.NewProgressSpinner("Removing Zarf package %s", p.cfg.PkgOpts.PackageSource) defer spinner.Stop() - // If components were provided; just remove the things we were asked to remove - requestedComponents := helpers.StringToSlice(p.cfg.PkgOpts.OptionalComponents) - partialRemove := len(requestedComponents) > 0 && requestedComponents[0] != "" - var packageName string // we do not want to allow removal of signed packages without a signature if there are remove actions @@ -48,26 +44,25 @@ func (p *Packager) Remove() (err error) { p.filterComponents() packageName = p.cfg.Pkg.Metadata.Name - // If we have package components check them for images, charts, manifests, etc - for _, component := range p.cfg.Pkg.Components { - // Flip requested based on if this is a partial removal - requested := !partialRemove + // Build a list of components to remove and determine if we need a cluster connection + componentsToRemove := []string{} + packageRequiresCluster := false - if slices.Contains(requestedComponents, component.Name) { - requested = true - } + // If components were provided; just remove the things we were asked to remove + p.forIncludedComponents(func(component types.ZarfComponent) error { + componentsToRemove = append(componentsToRemove, component.Name) - if requested { - if p.requiresCluster(component) { - requiresCluster = true - } + if requiresCluster(component) { + packageRequiresCluster = true } - } - // Get the secret for the deployed package + return nil + }) + + // Get or build the secret for the deployed package deployedPackage := &types.DeployedPackage{} - if requiresCluster { + if packageRequiresCluster { err = p.connectToCluster(cluster.DefaultTimeout) if err != nil { return err @@ -80,25 +75,19 @@ func (p *Packager) Remove() (err error) { // If we do not need the cluster, create a deployed components object based on the info we have deployedPackage.Name = packageName deployedPackage.Data = p.cfg.Pkg - if partialRemove { - for _, r := range requestedComponents { - deployedPackage.DeployedComponents = append(deployedPackage.DeployedComponents, types.DeployedComponent{Name: r}) - } - } else { - for _, c := range p.cfg.Pkg.Components { - deployedPackage.DeployedComponents = append(deployedPackage.DeployedComponents, types.DeployedComponent{Name: c.Name}) - } + for _, r := range componentsToRemove { + deployedPackage.DeployedComponents = append(deployedPackage.DeployedComponents, types.DeployedComponent{Name: r}) } } - for _, c := range helpers.Reverse(deployedPackage.DeployedComponents) { + for _, dc := range helpers.Reverse(deployedPackage.DeployedComponents) { // Only remove the component if it was requested or if we are removing the whole package - if partialRemove && !slices.Contains(requestedComponents, c.Name) { + if !slices.Contains(componentsToRemove, dc.Name) { continue } - if deployedPackage, err = p.removeComponent(deployedPackage, c, spinner); err != nil { - return fmt.Errorf("unable to remove the component '%s': %w", c.Name, err) + if deployedPackage, err = p.removeComponent(deployedPackage, dc, spinner); err != nil { + return fmt.Errorf("unable to remove the component '%s': %w", dc.Name, err) } } diff --git a/src/pkg/packager/sources/cluster.go b/src/pkg/packager/sources/cluster.go index a623b85b27..4718092474 100644 --- a/src/pkg/packager/sources/cluster.go +++ b/src/pkg/packager/sources/cluster.go @@ -16,13 +16,13 @@ import ( ) var ( - // veryify that ClusterSource implements PackageSource + // verify that ClusterSource implements PackageSource _ PackageSource = (*ClusterSource)(nil) ) // NewClusterSource creates a new cluster source. func NewClusterSource(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { - if !validate.IsLowercaseNumberHyphen(pkgOpts.PackageSource) { + if !validate.IsLowercaseNumberHyphenNoStartHyphen(pkgOpts.PackageSource) { return nil, fmt.Errorf("invalid package name %q", pkgOpts.PackageSource) } cluster, err := cluster.NewClusterWithWait(cluster.DefaultTimeout) diff --git a/src/pkg/packager/sources/new.go b/src/pkg/packager/sources/new.go index c0d6d68a4e..9cd8c07c1a 100644 --- a/src/pkg/packager/sources/new.go +++ b/src/pkg/packager/sources/new.go @@ -66,7 +66,8 @@ func New(pkgOpts *types.ZarfPackageOptions) (PackageSource, error) { if pkgOpts.Shasum != "" { pkgSrc = fmt.Sprintf("%s@sha256:%s", pkgSrc, pkgOpts.Shasum) } - remote, err := oci.NewOrasRemote(pkgSrc) + arch := config.GetArch() + remote, err := oci.NewOrasRemote(pkgSrc, oci.WithArch(arch)) if err != nil { return nil, err } diff --git a/src/pkg/packager/sources/new_test.go b/src/pkg/packager/sources/new_test.go index abcaace81b..b1cdea51bf 100644 --- a/src/pkg/packager/sources/new_test.go +++ b/src/pkg/packager/sources/new_test.go @@ -25,7 +25,7 @@ type source struct { } var sources = []source{ - {pkgSrc: "oci://ghcr.io/defenseunicorns/packages/init:1.0.0-amd64", srcType: "oci", source: ociS}, + {pkgSrc: "oci://ghcr.io/defenseunicorns/packages/init:1.0.0", srcType: "oci", source: ociS}, {pkgSrc: "sget://github.com/defenseunicorns/zarf-hello-world:x86", srcType: "sget", source: urlS}, {pkgSrc: "sget://defenseunicorns/zarf-hello-world:x86_64", srcType: "sget", source: urlS}, {pkgSrc: "https://github.com/defenseunicorns/zarf/releases/download/v1.0.0/zarf-init-amd64-v1.0.0.tar.zst", srcType: "https", source: urlS}, diff --git a/src/pkg/packager/sources/oci.go b/src/pkg/packager/sources/oci.go index 62e22a5f47..371c53a7b5 100644 --- a/src/pkg/packager/sources/oci.go +++ b/src/pkg/packager/sources/oci.go @@ -197,7 +197,8 @@ func (s *OCISource) Collect(dir string) (string, error) { spinner.Success() - isSkeleton := strings.HasSuffix(s.Repo().Reference.Reference, oci.SkeletonSuffix) + // TODO (@Noxsios) remove the suffix check at v1.0.0 + isSkeleton := pkg.Build.Architecture == "skeleton" || strings.HasSuffix(s.Repo().Reference.Reference, oci.SkeletonArch) name := NameFromMetadata(&pkg, isSkeleton) dstTarball := filepath.Join(dir, name) diff --git a/src/pkg/packager/sources/split.go b/src/pkg/packager/sources/split.go index 296b26deb2..1aade2eee5 100644 --- a/src/pkg/packager/sources/split.go +++ b/src/pkg/packager/sources/split.go @@ -40,9 +40,9 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { // Ensure the files are in order so they are appended in the correct order sort.Strings(fileList) - reassmbled := filepath.Join(dir, filepath.Base(strings.Replace(s.PackageSource, ".part000", "", 1))) + reassembled := filepath.Join(dir, filepath.Base(strings.Replace(s.PackageSource, ".part000", "", 1))) // Create the new package - pkgFile, err := os.Create(reassmbled) + pkgFile, err := os.Create(reassembled) if err != nil { return "", fmt.Errorf("unable to create new package file: %s", err) } @@ -87,7 +87,7 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } } - if err := utils.SHAsMatch(reassmbled, pkgData.Sha256Sum); err != nil { + if err := utils.SHAsMatch(reassembled, pkgData.Sha256Sum); err != nil { return "", fmt.Errorf("package integrity check failed: %w", err) } @@ -97,9 +97,9 @@ func (s *SplitTarballSource) Collect(dir string) (string, error) { } // communicate to the user that the package was reassembled - message.Infof("Reassembled package to: %q", reassmbled) + message.Infof("Reassembled package to: %q", reassembled) - return reassmbled, nil + return reassembled, nil } // LoadPackage loads a package from a split tarball. diff --git a/src/pkg/packager/sources/tarball.go b/src/pkg/packager/sources/tarball.go index e5b13a2749..e65d0b962a 100644 --- a/src/pkg/packager/sources/tarball.go +++ b/src/pkg/packager/sources/tarball.go @@ -45,7 +45,6 @@ func (s *TarballSource) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) pathsExtracted := []string{} - // Walk the package so that was can dynamically load a .tar or a .tar.zst without caring about filenames. err = archiver.Walk(s.PackageSource, func(f archiver.File) error { if f.IsDir() { return nil diff --git a/src/pkg/packager/sources/utils.go b/src/pkg/packager/sources/utils.go index 97337f4944..383cc262cd 100644 --- a/src/pkg/packager/sources/utils.go +++ b/src/pkg/packager/sources/utils.go @@ -92,6 +92,10 @@ func RenameFromMetadata(path string) (string, error) { return "", err } + if pkg.Metadata.Name == "" { + return "", fmt.Errorf("%q does not contain a zarf.yaml", path) + } + name := NameFromMetadata(&pkg, false) name = fmt.Sprintf("%s%s", name, ext) diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 2caa135907..fb3e20f6d3 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -15,6 +15,30 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) +// ReloadComponentTemplate appends ###ZARF_COMPONENT_NAME### for the component, assigns value, and reloads +// Any instance of ###ZARF_COMPONENT_NAME### within a component will be replaced with that components name +func ReloadComponentTemplate(component *types.ZarfComponent) error { + mappings := map[string]string{} + mappings[types.ZarfComponentName] = component.Name + err := utils.ReloadYamlTemplate(component, mappings) + if err != nil { + return err + } + return nil +} + +// ReloadComponentTemplatesInPackage appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads +func ReloadComponentTemplatesInPackage(zarfPackage *types.ZarfPackage) error { + // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value + for i := range zarfPackage.Components { + if err := ReloadComponentTemplate(&zarfPackage.Components[i]); err != nil { + return err + } + } + + return nil +} + // fillActiveTemplate handles setting the active variables and reloading the base template. func (p *Packager) fillActiveTemplate() error { templateMap := map[string]string{} @@ -54,7 +78,7 @@ func (p *Packager) fillActiveTemplate() error { } // update the component templates on the package - err := p.findComponentTemplatesAndReload() + err := ReloadComponentTemplatesInPackage(&p.cfg.Pkg) if err != nil { return err } @@ -126,21 +150,6 @@ func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoI } } -// findComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads -func (p *Packager) findComponentTemplatesAndReload() error { - // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value - for i, component := range p.cfg.Pkg.Components { - mappings := map[string]string{} - mappings[types.ZarfComponentName] = component.Name - err := utils.ReloadYamlTemplate(&p.cfg.Pkg.Components[i], mappings) - if err != nil { - return err - } - } - - return nil -} - // checkVariablePattern checks to see if a current variable is set to a value that matches its pattern func (p *Packager) checkVariablePattern(name, pattern string) error { if regexp.MustCompile(pattern).MatchString(p.cfg.SetVariableMap[name].Value) { diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index e7c991d3d8..ba38a31bf1 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -90,6 +90,10 @@ func (p *Packager) writeYaml() error { p.cfg.Pkg.Metadata.Architecture = p.arch p.cfg.Pkg.Build.Architecture = p.arch + if p.cfg.CreateOpts.IsSkeleton { + p.cfg.Pkg.Build.Architecture = "skeleton" + } + // Record the time of package creation. p.cfg.Pkg.Build.Timestamp = now.Format(time.RFC1123Z) @@ -107,6 +111,9 @@ func (p *Packager) writeYaml() error { deprecated.PluralizeSetVariable, } + // Record the flavor of Zarf used to build this package (if any). + p.cfg.Pkg.Build.Flavor = p.cfg.CreateOpts.Flavor + p.cfg.Pkg.Build.RegistryOverrides = p.cfg.CreateOpts.RegistryOverrides // Record the latest version of Zarf without breaking changes to the package structure. diff --git a/src/pkg/transform/image_test.go b/src/pkg/transform/image_test.go index 819f9f672e..2edc2e87ec 100644 --- a/src/pkg/transform/image_test.go +++ b/src/pkg/transform/image_test.go @@ -15,6 +15,7 @@ var imageRefs = []string{ "nginx:1.23.3", "defenseunicorns/zarf-agent:v0.22.1", "defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "ghcr.io/stefanprodan/podinfo:6.3.3", "registry1.dso.mil/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -33,6 +34,7 @@ func TestImageTransformHost(t *testing.T) { "gitlab.com/project/library/nginx:1.23.3-zarf-3793515731", "gitlab.com/project/defenseunicorns/zarf-agent:v0.22.1-zarf-4283503412", "gitlab.com/project/defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "gitlab.com/project/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "gitlab.com/project/stefanprodan/podinfo:6.3.3-zarf-2985051089", "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0-zarf-2003217571", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -56,6 +58,7 @@ func TestImageTransformHostWithoutChecksum(t *testing.T) { "gitlab.com/project/library/nginx:1.23.3", "gitlab.com/project/defenseunicorns/zarf-agent:v0.22.1", "gitlab.com/project/defenseunicorns/zarf-agent@sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de", + "gitlab.com/project/library/busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", "gitlab.com/project/stefanprodan/podinfo:6.3.3", "gitlab.com/project/ironbank/opensource/defenseunicorns/zarf/zarf-agent:v0.25.0", "gitlab.com/project/gitea/gitea:1.19.3-rootless-zarf-3431384023", @@ -79,6 +82,7 @@ func TestParseImageRef(t *testing.T) { {"docker.io/", "library/nginx", "1.23.3", ""}, {"docker.io/", "defenseunicorns/zarf-agent", "v0.22.1", ""}, {"docker.io/", "defenseunicorns/zarf-agent", "", "sha256:84605f731c6a18194794c51e70021c671ab064654b751aa57e905bce55be13de"}, + {"docker.io/", "library/busybox", "latest", "sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79"}, {"ghcr.io/", "stefanprodan/podinfo", "6.3.3", ""}, {"registry1.dso.mil/", "ironbank/opensource/defenseunicorns/zarf/zarf-agent", "v0.25.0", ""}, {"gitlab.com/", "project/gitea/gitea", "1.19.3-rootless-zarf-3431384023", ""}, @@ -89,13 +93,19 @@ func TestParseImageRef(t *testing.T) { require.NoError(t, err) tag := expectedResult[idx][2] digest := expectedResult[idx][3] - tagOrDigest := ":" + tag - if tag == "" { + var tagOrDigest string + var tagAndDigest string + if tag != "" { + tagOrDigest = ":" + tag + tagAndDigest = ":" + tag + } + if digest != "" { tagOrDigest = "@" + digest + tagAndDigest += "@" + digest } path := expectedResult[idx][1] name := expectedResult[idx][0] + path - reference := name + tagOrDigest + reference := name + tagAndDigest require.Equal(t, reference, img.Reference) require.Equal(t, name, img.Name) diff --git a/src/pkg/utils/helpers/misc.go b/src/pkg/utils/helpers/misc.go index bbe722606f..9b6fefbbd8 100644 --- a/src/pkg/utils/helpers/misc.go +++ b/src/pkg/utils/helpers/misc.go @@ -19,7 +19,7 @@ func Retry(fn func() error, retries int, delay time.Duration, logger func(format break } - logger("Encountered an error, retrying (%d/%d): %s", r+1, retries, err.Error()) + logger("Retrying (%d/%d): %s", r+1, retries, err.Error()) time.Sleep(delay) } diff --git a/src/pkg/utils/helpers/random.go b/src/pkg/utils/helpers/random.go new file mode 100644 index 0000000000..63e7bfa89c --- /dev/null +++ b/src/pkg/utils/helpers/random.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +import ( + "crypto/rand" +) + +// Very limited special chars for git / basic auth +// https://owasp.org/www-community/password-special-characters has complete list of safe chars. +const randomStringChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!~-" + +// RandomString generates a secure random string of the specified length. +func RandomString(length int) (string, error) { + bytes := make([]byte, length) + + if _, err := rand.Read(bytes); err != nil { + //message.Fatal(err, "unable to generate a random secret") + return "", err + } + + for i, b := range bytes { + bytes[i] = randomStringChars[b%byte(len(randomStringChars))] + } + + return string(bytes), nil +} diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 436e8b0155..c02ce6b55e 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -5,6 +5,7 @@ package utils import ( + "archive/tar" "bufio" "crypto/sha256" "fmt" @@ -16,6 +17,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" @@ -422,3 +424,64 @@ func SHAsMatch(path, expected string) error { } return nil } + +// CreateReproducibleTarballFromDir creates a tarball from a directory with stripped headers +func CreateReproducibleTarballFromDir(dirPath, dirPrefix, tarballPath string) error { + tb, err := os.Create(tarballPath) + if err != nil { + return fmt.Errorf("error creating tarball: %w", err) + } + defer tb.Close() + + tw := tar.NewWriter(tb) + defer tw.Close() + + // Walk through the directory and process each file + return filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Create a new header + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return fmt.Errorf("error creating tar header: %w", err) + } + + // Strip non-deterministic header data + header.ModTime = time.Time{} + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + + // Ensure the header's name is correctly set relative to the base directory + name, err := filepath.Rel(dirPath, filePath) + if err != nil { + return fmt.Errorf("error getting relative path: %w", err) + } + header.Name = filepath.Join(dirPrefix, name) + + // Write the header to the tarball + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("error writing header: %w", err) + } + + // If it's a file, write its content + if !info.IsDir() { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("error opening file: %w", err) + } + defer file.Close() + + if _, err := io.Copy(tw, file); err != nil { + return fmt.Errorf("error writing file to tarball: %w", err) + } + } + + return nil + }) +} diff --git a/src/pkg/utils/random.go b/src/pkg/utils/random.go deleted file mode 100644 index 7966cd765a..0000000000 --- a/src/pkg/utils/random.go +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package utils provides generic utility functions. -package utils - -import ( - "crypto/rand" - "fmt" - - "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/fatih/color" -) - -// Very limited special chars for git / basic auth -// https://owasp.org/www-community/password-special-characters has complete list of safe chars. -const randomStringChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!~-" - -// RandomString generates a secure random string of the specified length. -func RandomString(length int) string { - bytes := make([]byte, length) - - if _, err := rand.Read(bytes); err != nil { - message.Fatal(err, "unable to generate a random secret") - } - - for i, b := range bytes { - bytes[i] = randomStringChars[b%byte(len(randomStringChars))] - } - - return string(bytes) -} - -// First30last30 returns the source string that has been trimmed to 30 characters at the beginning and end. -func First30last30(s string) string { - if len(s) > 60 { - return s[0:27] + "..." + s[len(s)-26:] - } - - return s -} - -// ColorWrap changes a string to an ansi color code and appends the default color to the end -// preventing future characters from taking on the given color -func ColorWrap(str string, attr color.Attribute) string { - return fmt.Sprintf("\x1b[%dm%s\x1b[0m", attr, str) -} diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index a6448dcd49..2eaf6a6ed8 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -83,17 +83,21 @@ func TestUseCLI(t *testing.T) { t.Run("zarf prepare find-images --kube-version", func(t *testing.T) { t.Parallel() - // Test `zarf prepare find-images` on a chart that has a `kubeVersion` declaration greater than the default (v1.20.0) + controllerImageWithTag := "quay.io/jetstack/cert-manager-controller:v1.11.1" + controlImageWithSignature := "quay.io/jetstack/cert-manager-controller:sha256-4f1782c8316f34aae6b9ab823c3e6b7e6e4d92ec5dac21de6a17c3da44c364f1.sig" + + // Test `zarf prepare find-images` on a chart that has a `kubeVersion` declaration greater than the Helm default (v1.20.0) + // This should pass because we build Zarf specifying the kubeVersion value from the kubernetes client-go library instead stdOut, stdErr, err := e2e.Zarf("prepare", "find-images", "src/test/packages/00-kube-version-override") + require.NoError(t, err, stdOut, stdErr) + require.Contains(t, stdOut, controllerImageWithTag, "The chart image should be found by Zarf") + require.Contains(t, stdOut, controlImageWithSignature, "The image signature should be found by Zarf") + + // Test `zarf prepare find-images` with `--kube-version` specified and less than than the declared minimum (v1.21.0) + stdOut, stdErr, err = e2e.Zarf("prepare", "find-images", "--kube-version=v1.20.0", "src/test/packages/00-kube-version-override") require.Error(t, err, stdOut, stdErr) require.Contains(t, stdErr, "Problem rendering the helm template for https://charts.jetstack.io/", "The kubeVersion declaration should prevent this from templating") require.Contains(t, stdErr, "following charts had errors: [https://charts.jetstack.io/]", "Zarf should print an ending error message") - - // Test `zarf prepare find-images` with `--kube-version` specified and greater than the declared minimum (v1.21.0) - stdOut, stdErr, err = e2e.Zarf("prepare", "find-images", "--kube-version=v1.22.0", "src/test/packages/00-kube-version-override") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdOut, "quay.io/jetstack/cert-manager-controller:v1.11.1", "The chart image should be found by Zarf") - require.Contains(t, stdOut, "quay.io/jetstack/cert-manager-controller:sha256-4f1782c8316f34aae6b9ab823c3e6b7e6e4d92ec5dac21de6a17c3da44c364f1.sig", "The image signature should be found by Zarf") }) t.Run("zarf deploy should fail when given a bad component input", func(t *testing.T) { @@ -104,6 +108,23 @@ func TestUseCLI(t *testing.T) { require.Error(t, err) }) + t.Run("zarf deploy should return a warning when no components are deployed", func(t *testing.T) { + t.Parallel() + _, _, err := e2e.Zarf("package", "create", "src/test/packages/00-no-components", "-o=build", "--confirm") + require.NoError(t, err) + path := fmt.Sprintf("build/zarf-package-no-components-%s.tar.zst", e2e.Arch) + + // Test that excluding all components with a leading dash results in a warning + _, stdErr, err := e2e.Zarf("package", "deploy", path, "--components=-deselect-me", "--confirm") + require.NoError(t, err) + require.Contains(t, stdErr, "No components were selected for deployment") + + // Test that excluding still works even if a wildcard is given + _, stdErr, err = e2e.Zarf("package", "deploy", path, "--components=*,-deselect-me", "--confirm") + require.NoError(t, err) + require.NotContains(t, stdErr, "DESELECT-ME COMPONENT") + }) + t.Run("changing log level", func(t *testing.T) { t.Parallel() // Test that changing the log level actually applies the requested level @@ -119,6 +140,16 @@ func TestUseCLI(t *testing.T) { require.Error(t, err, stdOut, stdErr) }) + t.Run("zarf package to test bad remote images", func(t *testing.T) { + _, stdErr, err := e2e.Zarf("package", "create", "src/test/packages/00-remote-pull-fail", "--confirm") + // expecting zarf to have an error and output to stderr + require.Error(t, err) + // Make sure we print the get request error (only look for GET since the actual error changes based on login status) + require.Contains(t, stdErr, "failed to find the manifest on a remote: GET") + // And the docker error + require.Contains(t, stdErr, "response from daemon: No such image") + }) + t.Run("zarf package to test archive path", func(t *testing.T) { t.Parallel() stdOut, stdErr, err := e2e.Zarf("package", "create", "packages/distros/eks", "--confirm") diff --git a/src/test/e2e/05_multi_part_test.go b/src/test/e2e/05_multi_part_test.go deleted file mode 100644 index f9c48ecb4a..0000000000 --- a/src/test/e2e/05_multi_part_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package test provides e2e tests for Zarf. -package test - -import ( - "fmt" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestMultiPartPackage(t *testing.T) { - t.Log("E2E: Multi-part package") - - var ( - createPath = "src/test/packages/05-multi-part" - deployPath = fmt.Sprintf("zarf-package-multi-part-%s.tar.zst.part000", e2e.Arch) - outputFile = "multi-part-demo.dat" - ) - - e2e.CleanFiles(deployPath, outputFile) - - // Create the package with a max size of 1MB - stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=1", "--confirm") - require.NoError(t, err, stdOut, stdErr) - - list, err := filepath.Glob("zarf-package-multi-part-*") - require.NoError(t, err) - // Length is 7 because there are 6 parts and 1 manifest - require.Len(t, list, 7) - - stdOut, stdErr, err = e2e.Zarf("package", "deploy", deployPath, "--confirm") - require.NoError(t, err, stdOut, stdErr) - - // Verify the package was deployed - require.FileExists(t, outputFile) - - e2e.CleanFiles(deployPath, outputFile) -} diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go new file mode 100644 index 0000000000..a473fed1ec --- /dev/null +++ b/src/test/e2e/05_tarball_test.go @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package test provides e2e tests for Zarf. +package test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/require" +) + +func TestMultiPartPackage(t *testing.T) { + t.Log("E2E: Multi-part package") + + var ( + createPath = "src/test/packages/05-multi-part" + deployPath = fmt.Sprintf("zarf-package-multi-part-%s.tar.zst.part000", e2e.Arch) + outputFile = "multi-part-demo.dat" + ) + + e2e.CleanFiles(deployPath, outputFile) + + // Create the package with a max size of 1MB + stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--max-package-size=1", "--confirm") + require.NoError(t, err, stdOut, stdErr) + + list, err := filepath.Glob("zarf-package-multi-part-*") + require.NoError(t, err) + // Length is 7 because there are 6 parts and 1 manifest + require.Len(t, list, 7) + + stdOut, stdErr, err = e2e.Zarf("package", "deploy", deployPath, "--confirm") + require.NoError(t, err, stdOut, stdErr) + + // Verify the package was deployed + require.FileExists(t, outputFile) + + e2e.CleanFiles(deployPath, outputFile) +} + +func TestReproducibleTarballs(t *testing.T) { + t.Log("E2E: Reproducible tarballs") + + var ( + createPath = filepath.Join("examples", "dos-games") + tmp = t.TempDir() + tb = filepath.Join(tmp, fmt.Sprintf("zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch)) + unpack1 = filepath.Join(tmp, "unpack1") + unpack2 = filepath.Join(tmp, "unpack2") + ) + + stdOut, stdErr, err := e2e.Zarf("package", "create", createPath, "--confirm", "--output", tmp) + require.NoError(t, err, stdOut, stdErr) + + stdOut, stdErr, err = e2e.Zarf("tools", "archiver", "decompress", tb, unpack1) + require.NoError(t, err, stdOut, stdErr) + + var pkg1 types.ZarfPackage + err = utils.ReadYaml(filepath.Join(unpack1, layout.ZarfYAML), &pkg1) + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(unpack1, layout.Checksums)) + require.NoError(t, err) + checksums1 := string(b) + + e2e.CleanFiles(unpack1, tb) + + stdOut, stdErr, err = e2e.Zarf("package", "create", createPath, "--confirm", "--output", tmp) + require.NoError(t, err, stdOut, stdErr) + + stdOut, stdErr, err = e2e.Zarf("tools", "archiver", "decompress", tb, unpack2) + require.NoError(t, err, stdOut, stdErr) + + var pkg2 types.ZarfPackage + err = utils.ReadYaml(filepath.Join(unpack2, layout.ZarfYAML), &pkg2) + require.NoError(t, err) + + b, err = os.ReadFile(filepath.Join(unpack2, layout.Checksums)) + require.NoError(t, err) + checksums2 := string(b) + + message.PrintDiff(checksums1, checksums2) + + require.Equal(t, pkg1.Metadata.AggregateChecksum, pkg2.Metadata.AggregateChecksum) +} diff --git a/src/test/e2e/06_create_sbom_test.go b/src/test/e2e/06_create_sbom_test.go index 1af4adf739..88e153860b 100644 --- a/src/test/e2e/06_create_sbom_test.go +++ b/src/test/e2e/06_create_sbom_test.go @@ -54,9 +54,9 @@ func TestCreateSBOM(t *testing.T) { _, err = os.ReadFile(filepath.Join(sbomPath, "dos-games", "sbom-viewer-docker.io_defenseunicorns_zarf-game_multi-tile-dark.html")) require.NoError(t, err) // Test that the init package generates the SBOMs we expect (images + component files) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.19.3-rootless.html")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-docker.io_gitea_gitea_1.21.2-rootless.html")) require.NoError(t, err) - _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.19.3-rootless.json")) + _, err = os.ReadFile(filepath.Join(sbomPath, "init", "docker.io_gitea_gitea_1.21.2-rootless.json")) require.NoError(t, err) _, err = os.ReadFile(filepath.Join(sbomPath, "init", "sbom-viewer-zarf-component-k3s.html")) require.NoError(t, err) diff --git a/src/test/e2e/10_component_flavor_test.go b/src/test/e2e/10_component_flavor_test.go index e94a8f1efb..3ae8cfc234 100644 --- a/src/test/e2e/10_component_flavor_test.go +++ b/src/test/e2e/10_component_flavor_test.go @@ -31,7 +31,7 @@ func (suite *FlavorSuite) SetupSuite() { suite.Assertions = require.New(suite.T()) // Setup the example package path after e2e has been initialized - flavorExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-package-flavors-%s.tar.zst", e2e.Arch)) + flavorExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-package-flavors-%s-1.0.0.tar.zst", e2e.Arch)) } func (suite *FlavorSuite) TearDownSuite() { diff --git a/src/test/e2e/12_lint_test.go b/src/test/e2e/12_lint_test.go index 98257fdc89..cf132c4068 100644 --- a/src/test/e2e/12_lint_test.go +++ b/src/test/e2e/12_lint_test.go @@ -1,31 +1,60 @@ package test import ( + "fmt" + "os" "path/filepath" "testing" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/stretchr/testify/require" ) func TestLint(t *testing.T) { t.Log("E2E: Lint") + t.Run("zarf test lint success", func(t *testing.T) { + t.Log("E2E: Test lint on schema success") + + // This runs lint on the zarf.yaml in the base directory of the repo + _, _, err := e2e.Zarf("dev", "lint") + require.NoError(t, err, "Expect no error here because the yaml file is following schema") + }) + t.Run("zarf test lint fail", func(t *testing.T) { t.Log("E2E: Test lint on schema fail") - path := filepath.Join("src", "test", "packages", "12-lint") - _, stderr, err := e2e.Zarf("prepare", "lint", path) + testPackagePath := filepath.Join("src", "test", "packages", "12-lint") + configPath := filepath.Join(testPackagePath, "zarf-config.toml") + os.Setenv("ZARF_CONFIG", configPath) + _, stderr, err := e2e.Zarf("dev", "lint", testPackagePath, "-f", "good-flavor") require.Error(t, err, "Require an exit code since there was warnings / errors") - require.Contains(t, stderr, ".components.[0].import: Additional property not-path is not allowed") - require.Contains(t, stderr, ".components.[2].import.path: Will not resolve ZARF_PKG_TMPL_* variables") - require.Contains(t, stderr, ".variables: Invalid type. Expected: array, given: null") - }) + strippedStderr := e2e.StripMessageFormatting(stderr) - t.Run("zarf test lint success", func(t *testing.T) { - t.Log("E2E: Test lint on schema success") + key := "WHATEVER_IMAGE" + require.Contains(t, strippedStderr, lang.UnsetVarLintWarning) + require.Contains(t, strippedStderr, fmt.Sprintf(lang.PkgValidateTemplateDeprecation, key, key, key)) + require.Contains(t, strippedStderr, ".components.[2].repos.[0] | Unpinned repository") + require.Contains(t, strippedStderr, ".metadata | Additional property description1 is not allowed") + require.Contains(t, strippedStderr, ".components.[0].import | Additional property not-path is not allowed") + // Testing the import / compose on lint is working + require.Contains(t, strippedStderr, ".components.[1].images.[0] | Image not pinned with digest - registry.com:9001/whatever/image:latest") + // Testing import / compose + variables are working + require.Contains(t, strippedStderr, ".components.[2].images.[3] | Image not pinned with digest - busybox:latest") + require.Contains(t, strippedStderr, ".components.[3].import.path | Zarf does not evaluate variables at component.x.import.path - ###ZARF_PKG_TMPL_PATH###") + // Testing OCI imports get linted + require.Contains(t, strippedStderr, ".components.[0].images.[0] | Image not pinned with digest - defenseunicorns/zarf-game:multi-tile-dark") + // Testing a bad path leads to a finding in lint + require.Contains(t, strippedStderr, fmt.Sprintf(".components.[3].import.path | open %s", filepath.Join("###ZARF_PKG_TMPL_PATH###", "zarf.yaml"))) + + // Check flavors + require.NotContains(t, strippedStderr, "image-in-bad-flavor-component:unpinned") + require.Contains(t, strippedStderr, "image-in-good-flavor-component:unpinned") + + // Check reported filepaths + require.Contains(t, strippedStderr, "Linting package \"dos-games\" at oci://🦄/dos-games:1.0.0") + require.Contains(t, strippedStderr, fmt.Sprintf("Linting package \"lint\" at %s", testPackagePath)) - // This runs lint on the zarf.yaml in the base directory of the repo - _, _, err := e2e.Zarf("prepare", "lint") - require.NoError(t, err, "Expect no error here because the yaml file is following schema") }) + } diff --git a/src/test/e2e/22_git_and_gitops_test.go b/src/test/e2e/22_git_and_gitops_test.go index e09280a5b4..d9c4b572d4 100644 --- a/src/test/e2e/22_git_and_gitops_test.go +++ b/src/test/e2e/22_git_and_gitops_test.go @@ -31,8 +31,8 @@ func TestGit(t *testing.T) { path := fmt.Sprintf("build/zarf-package-git-data-test-%s-1.0.0.tar.zst", e2e.Arch) defer e2e.CleanFiles(path) - // Deploy the git data example - stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--confirm") + // Deploy the git data example (with component globbing to test that as well) + stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--components=full-repo,specific-*", "--confirm") require.NoError(t, err, stdOut, stdErr) c, err := cluster.NewCluster() @@ -77,7 +77,7 @@ func testGitServerReadOnly(t *testing.T, gitURL string) { // Get the repo as the readonly user repoName := "zarf-public-test-2469062884" getRepoRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s", gitURL, state.GitServer.PushUsername, repoName), nil) - getRepoResponseBody, err := gitCfg.DoHTTPThings(getRepoRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) + getRepoResponseBody, _, err := gitCfg.DoHTTPThings(getRepoRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) require.NoError(t, err) // Make sure the only permissions are pull (read) @@ -100,7 +100,7 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Get the Zarf repo tag repoTag := "v0.0.1" getRepoTagsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/tags/%s", gitURL, config.ZarfGitPushUser, repoName, repoTag), nil) - getRepoTagsResponseBody, err := gitCfg.DoHTTPThings(getRepoTagsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) + getRepoTagsResponseBody, _, err := gitCfg.DoHTTPThings(getRepoTagsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) require.NoError(t, err) // Make sure the pushed tag exists @@ -111,7 +111,7 @@ func testGitServerTagAndHash(t *testing.T, gitURL string) { // Get the Zarf repo commit repoHash := "01a23218923f24194133b5eb11268cf8d73ff1bb" getRepoCommitsRequest, _ := http.NewRequest("GET", fmt.Sprintf("%s/api/v1/repos/%s/%s/git/commits/%s", gitURL, config.ZarfGitPushUser, repoName, repoHash), nil) - getRepoCommitsResponseBody, err := gitCfg.DoHTTPThings(getRepoCommitsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) + getRepoCommitsResponseBody, _, err := gitCfg.DoHTTPThings(getRepoCommitsRequest, config.ZarfGitReadUser, state.GitServer.PullPassword) require.NoError(t, err) require.Contains(t, string(getRepoCommitsResponseBody), repoHash) } diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 5a6e35d067..55a038715b 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -58,7 +58,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_0_Publish() { suite.Contains(stdErr, "Published "+ref) // Pull the package via OCI. - stdOut, stdErr, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1-"+e2e.Arch, "--insecure") + stdOut, stdErr, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1", "--insecure") suite.NoError(err, stdOut, stdErr) // Publish w/ package missing `metadata.version` field. @@ -71,8 +71,17 @@ func (suite *PublishDeploySuiteTestSuite) Test_0_Publish() { stdOut, stdErr, err = e2e.Zarf("package", "create", dir, "-o", "oci://"+ref, "--insecure", "--oci-concurrency=5", "--confirm") suite.NoError(err, stdOut, stdErr) + // Inline publish flavor. + dir = filepath.Join("examples", "package-flavors") + stdOut, stdErr, err = e2e.Zarf("package", "create", dir, "-o", "oci://"+ref, "--flavor", "oracle-cookie-crunch", "--insecure", "--confirm") + suite.NoError(err, stdOut, stdErr) + + // Inspect published flavor. + stdOut, stdErr, err = e2e.Zarf("package", "inspect", "oci://"+ref+"/package-flavors:1.0.0-oracle-cookie-crunch", "--insecure") + suite.NoError(err, stdOut, stdErr) + // Inspect the published package. - stdOut, stdErr, err = e2e.Zarf("package", "inspect", "oci://"+ref+"/helm-charts:0.0.1-"+e2e.Arch, "--insecure") + stdOut, stdErr, err = e2e.Zarf("package", "inspect", "oci://"+ref+"/helm-charts:0.0.1", "--insecure") suite.NoError(err, stdOut, stdErr) } @@ -81,7 +90,7 @@ func (suite *PublishDeploySuiteTestSuite) Test_1_Deploy() { // Build the fully qualified reference. suite.Reference.Repository = "helm-charts" - suite.Reference.Reference = fmt.Sprintf("0.0.1-%s", e2e.Arch) + suite.Reference.Reference = "0.0.1" ref := suite.Reference.String() // Deploy the package via OCI. @@ -119,22 +128,17 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { e2e.SetupDockerRegistry(t, dstRegistryPort) defer e2e.TeardownRegistry(t, dstRegistryPort) - ctx := context.TODO() - - src, err := oci.NewOrasRemote(ref) + src, err := oci.NewOrasRemote(ref, oci.WithPlainHTTP(true), oci.WithArch(e2e.Arch)) suite.NoError(err) - src.WithInsecureConnection(true) - src.WithContext(ctx) - dst, err := oci.NewOrasRemote(dstRef) + dst, err := oci.NewOrasRemote(dstRef, oci.WithPlainHTTP(true), oci.WithArch(e2e.Arch)) suite.NoError(err) - dst.WithInsecureConnection(true) - dst.WithContext(ctx) reg, err := remote.NewRegistry(strings.Split(dstRef, "/")[0]) suite.NoError(err) reg.PlainHTTP = true attempt := 0 + ctx := context.TODO() for attempt <= 5 { err = reg.Ping(ctx) if err == nil { diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index e0cae59427..4ad8769d8e 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -75,16 +75,16 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.NoError(err) suite.Contains(stdErr, "Published "+ref) - _, _, err = e2e.Zarf("package", "inspect", "oci://"+ref+"/import-everything:0.0.1-skeleton", "--insecure") + _, _, err = e2e.Zarf("package", "inspect", "oci://"+ref+"/import-everything:0.0.1", "--insecure", "-a", "skeleton") suite.NoError(err) - _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/import-everything:0.0.1-skeleton", "-o", "build", "--insecure") + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/import-everything:0.0.1", "-o", "build", "--insecure", "-a", "skeleton") suite.NoError(err) - _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1-skeleton", "-o", "build", "--insecure") + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1", "-o", "build", "--insecure", "-a", "skeleton") suite.NoError(err) - _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-min:2.10.0-skeleton", "-o", "build", "--insecure") + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-min:2.10.0", "-o", "build", "--insecure", "-a", "skeleton") suite.NoError(err) } diff --git a/src/test/e2e/99_yolo_test.go b/src/test/e2e/99_yolo_test.go index 1f10634bc9..a4044c53a9 100644 --- a/src/test/e2e/99_yolo_test.go +++ b/src/test/e2e/99_yolo_test.go @@ -47,3 +47,17 @@ func TestYOLOMode(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "remove", "yolo", "--confirm") require.NoError(t, err, stdOut, stdErr) } + +func TestDevDeploy(t *testing.T) { + // Don't run this test in appliance mode + if e2e.ApplianceMode { + return + } + e2e.SetupWithCluster(t) + + stdOut, stdErr, err := e2e.Zarf("dev", "deploy", "examples/dos-games") + require.NoError(t, err, stdOut, stdErr) + + stdOut, stdErr, err = e2e.Zarf("package", "remove", "dos-games", "--confirm") + require.NoError(t, err, stdOut, stdErr) +} diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index aa078cc401..82cb1b2d33 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -11,6 +11,7 @@ import ( "testing" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/test" ) @@ -32,6 +33,9 @@ func TestMain(m *testing.M) { // K3d use the intern package, which requires this to be set in go 1.19 os.Setenv("ASSUME_NO_MOVING_GC_UNSAFE_RISK_IT_WITH", "go1.19") + // Set the log level to trace for when we call Zarf functions internally + message.SetLogLevel(message.TraceLevel) + retCode, err := doAllTheThings(m) if err != nil { fmt.Println(err) //nolint:forbidigo diff --git a/src/test/packages/00-no-components/zarf.yaml b/src/test/packages/00-no-components/zarf.yaml new file mode 100644 index 0000000000..78d89b650f --- /dev/null +++ b/src/test/packages/00-no-components/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfPackageConfig +metadata: + name: no-components + +components: +- name: deselect-me + default: true + +- name: optional diff --git a/src/test/packages/00-remote-pull-fail/zarf.yaml b/src/test/packages/00-remote-pull-fail/zarf.yaml new file mode 100644 index 0000000000..f44d99d4b1 --- /dev/null +++ b/src/test/packages/00-remote-pull-fail/zarf.yaml @@ -0,0 +1,8 @@ +kind: ZarfPackageConfig +metadata: + name: doesnotexist +components: + - name: doesnotexist-docker + required: true + images: + - ghcr.io/defenseunicorns/doesnotexist:1.3.3.7 diff --git a/src/test/packages/12-lint/linted-import/zarf.yaml b/src/test/packages/12-lint/linted-import/zarf.yaml new file mode 100644 index 0000000000..4a99f73e96 --- /dev/null +++ b/src/test/packages/12-lint/linted-import/zarf.yaml @@ -0,0 +1,23 @@ +kind: ZarfPackageConfig +metadata: + name: linted-import + description: Testing bad yaml imported + +variables: + - name: BUSYBOX_IMAGE + description: "whatever" + +components: + - name: dont-care + + - name: import-test + images: + - registry.com:9001/whatever/image:latest + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### + - busybox:###ZARF_PKG_TMPL_UNSET### + + - name: oci-games-url + import: + url: oci://🦄/dos-games:1.0.0 + name: baseline diff --git a/src/test/packages/12-lint/zarf-config.toml b/src/test/packages/12-lint/zarf-config.toml new file mode 100644 index 0000000000..1d1c57f33e --- /dev/null +++ b/src/test/packages/12-lint/zarf-config.toml @@ -0,0 +1,3 @@ +[package.create.set] +BUSYBOX_IMAGE = "latest" +PATH = "linted-import" diff --git a/src/test/packages/12-lint/zarf.yaml b/src/test/packages/12-lint/zarf.yaml index 7c026a15b7..980fa78eeb 100644 --- a/src/test/packages/12-lint/zarf.yaml +++ b/src/test/packages/12-lint/zarf.yaml @@ -1,11 +1,8 @@ -kind: ZarfInitConfig +kind: ZarfPackageConfig metadata: - name: init + name: lint description1: Testing bad yaml - -variables: - components: - name: first-test-component import: @@ -13,12 +10,47 @@ components: - name: import-test import: - path: 123123 + path: linted-import - - name: import-test + - name: full-repo + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + - https://dev.azure.com/defenseunicorns/zarf-public-test/_git/zarf-public-test@v0.0.1 + - https://gitlab.com/gitlab-org/build/omnibus-mirror/pcre2/-/tree/vreverse?ref_type=heads + images: + - registry.com:9001/whatever/image:1.0.0 + - busybox@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79 + - busybox:###ZARF_PKG_VAR_WHATEVER_IMAGE### + - busybox:###ZARF_PKG_TMPL_BUSYBOX_IMAGE### + - ubuntu:###ZARF_PKG_TMPL_UBUNTU_IMAGE### + files: + - source: https://github.com/k3s-io/k3s/releases/download/v1.28.2+k3s1/k3s + shasum: 2f041d37a2c6d54d53e106e1c7713bc48f806f3919b0d9e092f5fcbdc55b41cf + target: src/ + - source: file-without-shasum.txt + target: src/ + + - name: import import: - path: "###ZARF_PKG_TMPL_ZEBRA###" + path: "###ZARF_PKG_TMPL_PATH###" - - name: import-url + - name: oci-games-url import: - url: "oci://###ZARF_PKG_TMPL_ZEBRA###" + url: oci://🦄/dos-games:1.0.0 + name: baseline + + - name: oci-games-url + import: + path: linted-import + + - name: import-bad-flavor + only: + flavor: bad-flavor + images: + - image-in-bad-flavor-component:unpinned + + - name: import-good-flavor + only: + flavor: good-flavor + images: + - image-in-good-flavor-component:unpinned diff --git a/src/test/packages/28-helm-no-wait/zarf.yaml b/src/test/packages/28-helm-no-wait/zarf.yaml index 86eac186fd..177fe6792a 100644 --- a/src/test/packages/28-helm-no-wait/zarf.yaml +++ b/src/test/packages/28-helm-no-wait/zarf.yaml @@ -4,7 +4,7 @@ metadata: description: Deploys a pod which never becomes ready components: - - name: zarf-helm-no-wait + - name: helm-no-wait required: true manifests: - name: never-ready diff --git a/src/test/packages/51-import-everything/inception/zarf.yaml b/src/test/packages/51-import-everything/inception/zarf.yaml index 297f4b8b41..7de7d43d81 100644 --- a/src/test/packages/51-import-everything/inception/zarf.yaml +++ b/src/test/packages/51-import-everything/inception/zarf.yaml @@ -8,24 +8,24 @@ components: - name: import-component-local required: true import: - url: oci://localhost:555/import-everything:0.0.1-skeleton + url: oci://localhost:555/import-everything:0.0.1 - name: import-component-oci required: true import: - url: oci://localhost:555/import-everything:0.0.1-skeleton + url: oci://localhost:555/import-everything:0.0.1 - name: import-big-bang required: true import: - url: oci://localhost:555/import-everything:0.0.1-skeleton + url: oci://localhost:555/import-everything:0.0.1 - name: file-imports required: true import: - url: oci://localhost:555/import-everything:0.0.1-skeleton + url: oci://localhost:555/import-everything:0.0.1 - name: local-chart-import required: true import: - url: oci://localhost:555/import-everything:0.0.1-skeleton + url: oci://localhost:555/import-everything:0.0.1 diff --git a/src/test/packages/51-import-everything/oci-import/zarf.yaml b/src/test/packages/51-import-everything/oci-import/zarf.yaml index 6c27000904..42d4b79901 100644 --- a/src/test/packages/51-import-everything/oci-import/zarf.yaml +++ b/src/test/packages/51-import-everything/oci-import/zarf.yaml @@ -10,4 +10,4 @@ components: required: false import: name: demo-helm-charts - url: oci://localhost:555/helm-charts:0.0.1-skeleton + url: oci://localhost:555/helm-charts:0.0.1 diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index 5807c1431e..b69ef73086 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -27,7 +27,7 @@ components: required: false import: name: bigbang - url: oci://localhost:555/big-bang-min:2.10.0-skeleton + url: oci://localhost:555/big-bang-min:2.10.0 # Test file imports including cosignKeyPath - name: file-imports diff --git a/src/types/component.go b/src/types/component.go index 36fa1d529c..f63e6ab2a7 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -13,7 +13,7 @@ import ( // ZarfComponent is the primary functional grouping of assets to deploy by Zarf. type ZarfComponent struct { // Name is the unique identifier for this component - Name string `json:"name" jsonschema:"description=The name of the component,pattern=^[a-z0-9\\-]+$"` + Name string `json:"name" jsonschema:"description=The name of the component,pattern=^[a-z0-9\\-]*[a-z0-9]$"` // Description is a message given to a user when deciding to enable this component or not Description string `json:"description,omitempty" jsonschema:"description=Message to include during package deploy describing the purpose of this component"` diff --git a/src/types/package.go b/src/types/package.go index 6b44fe8a16..0ec8f58a23 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -26,7 +26,7 @@ type ZarfPackage struct { // ZarfMetadata lists information about the current ZarfPackage. type ZarfMetadata struct { - Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]+$"` + Name string `json:"name" jsonschema:"description=Name to identify this Zarf package,pattern=^[a-z0-9\\-]*[a-z0-9]$"` Description string `json:"description,omitempty" jsonschema:"description=Additional information about this package"` Version string `json:"version,omitempty" jsonschema:"description=Generic string set by a package author to track the package version (Note: ZarfInitConfigs will always be versioned to the CLIVersion they were created with)"` URL string `json:"url,omitempty" jsonschema:"description=Link to package information when online"` @@ -53,6 +53,7 @@ type ZarfBuildData struct { RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` + Flavor string `json:"flavor,omitempty" jsonschema:"description=The flavor of Zarf used to build this package"` } // ZarfPackageVariable are variables that can be used to dynamically template K8s resources. diff --git a/src/types/runtime.go b/src/types/runtime.go index ee41ec8bc2..3878bb0aef 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -4,7 +4,9 @@ // Package types contains all the types used by Zarf. package types -import "time" +import ( + "time" +) const ( // RawVariableType is the default type for a Zarf package variable @@ -109,6 +111,8 @@ type ZarfCreateOptions struct { DifferentialData DifferentialData `json:"differential" jsonschema:"description=A package's differential images and git repositories from a referenced previously built package"` RegistryOverrides map[string]string `json:"registryOverrides" jsonschema:"description=A map of domains to override on package create when pulling images"` Flavor string `json:"flavor" jsonschema:"description=An optional variant that controls which components will be included in a package"` + IsSkeleton bool `json:"isSkeleton" jsonschema:"description=Whether to create a skeleton package"` + NoYOLO bool `json:"noYOLO" jsonschema:"description=Whether to create a YOLO package"` } // ZarfSplitPackageData contains info about a split package. diff --git a/zarf-config.toml b/zarf-config.toml index 5f0f1b5357..2010031c92 100644 --- a/zarf-config.toml +++ b/zarf-config.toml @@ -15,5 +15,4 @@ registry_image = 'library/registry' registry_image_tag = '2.8.3' # The image reference to use for the optional git-server Zarf deploys -gitea_image = 'gitea/gitea' -gitea_server_version = '1.19.3' +gitea_image = 'gitea/gitea:1.21.2-rootless' diff --git a/zarf.schema.json b/zarf.schema.json index d8d1b5c3ac..d849c1b39a 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -135,6 +135,10 @@ "lastNonBreakingVersion": { "type": "string", "description": "The minimum version of Zarf that does not have breaking package structure changes" + }, + "flavor": { + "type": "string", + "description": "The flavor of Zarf used to build this package" } }, "additionalProperties": false, @@ -207,7 +211,7 @@ ], "properties": { "name": { - "pattern": "^[a-z0-9\\-]+$", + "pattern": "^[a-z0-9\\-]*[a-z0-9]$", "type": "string", "description": "The name of the component" }, @@ -832,7 +836,7 @@ ], "properties": { "name": { - "pattern": "^[a-z0-9\\-]+$", + "pattern": "^[a-z0-9\\-]*[a-z0-9]$", "type": "string", "description": "Name to identify this Zarf package" },