Support for Air-gapped Image building

projects/aws/image-builder/README.md

@@ -213,4 +213,65 @@ sudo snap install yq
 5. Run the image builder tool for appropriate release channel
 ```
 image-builder build --os ubuntu --hypervisor nutanix --nutanix-config <path to above json file> --release-channel <release channel, ex 1-23>
-```
+```
+
+### Air Gapped Image Building
+Image builder only supports building Ubuntu in an airgapped mode for now.
+
+1. Create the config json file for respective provider and make sure to include the fields required for airgapped building.
+   An example of baremetal config json for ubuntu airgapped builds are below
+   ```
+   {
+      "eksa_build_tooling_repo_url": "https://internal-repos/eks-anywhere-build-tooling.git",
+      "image_builder_repo_url": "https://internal-repos/image-builder.git",
+      "private_artifacts_eksd_fqdn": "http://artifactory:8081/artifactory",
+      "private_artifacts_eksa_fqdn": "http://artifactory:8081/artifactory/EKS-A",
+      "extra_repos": "/home/airgapped/sources.list",
+      "disable_public_repos": "true",
+      "iso_url": "http://artifactory:8081/artifactory/EKS-A/ISO/ubuntu-20.04.1-legacy-server-amd64.iso",
+      "iso_checksum": "f11bda2f2caed8f420802b59f382c25160b114ccc665dbac9c5046e7fceaced2",
+      "iso_checksum_type": "sha256"
+    }
+   ```
+2. Install pre-requisites required for image builder in the environment or admin machine.
+   1. Packer version 1.8.7 
+   2. Ansible version 2.11.5
+   3. Packer provisioner goss version 3.1.4
+   4. jq
+   5. yq
+   6. unzip 
+   7. make 
+   8. python3-pip
+3. From an environment with internet access run the following command to generate the manifest tarball
+    ```
+   image-builder download manifests
+    ```
+   This will download a eks-a-manifests.tar in the current working directory. This tarball is required for airgapped building.
+4. Replicate all the required EKS-D and EKS-A artifacts to the internal artifacts server like artifactory.
+   Required artifacts are as follows
+   EKS-D amd64 artifacts for specific release branch
+   1. kube-apiserver.tar
+   2. kube-scheduler.tar
+   3. kube-proxy.tar
+   4. kube-controller-manager.tar
+   5. etcd.tar
+   6. coredns.tar
+   7. pause.tar
+   8. kubectl
+   9. kubeadm
+   10. kubelet
+   11. etcd-linux-amd64-v<version>.tar.gz
+   12. cni-plugins-linux-amd64-v<version>.tar.gz
+
+
+   EKS-A amd64 artifacts for specific release bundle version
+   1. containerd-linux-amd64.tar.gz
+   2. etcdadm-linux-amd64.tar.gz
+   3. cri-tools-amd64.tar.gz
+
+   In addition to these EKS-D and EKS-A artifacts please ensure the base ubuntu iso is also hosted internally.
+
+5. Run image builder in airgapped mode
+   ```
+   image-builder build --os ubuntu --hypervisor baremetal --release-channel 1-27 --air-gapped --baremetal-config baremetal.json --manifest-tarball eks-a-manifests.tar
+   ```

projects/aws/image-builder/builder/builder.go

@@ -22,8 +22,28 @@ func (b *BuildOptions) BuildImage() {
 	if err != nil {
 		log.Fatalf("Error retrieving current working directory: %v", err)
 	}
+	if b.AirGapped {
+		var eksDArtifactsDomain, eksAArtifactsDomain string
+		switch b.Hypervisor {
+		case VSphere:
+			eksDArtifactsDomain = b.VsphereConfig.PrivateServerEksDDomainUrl
+			eksAArtifactsDomain = b.VsphereConfig.PrivateServerEksADomainUrl
+		case Baremetal:
+			eksDArtifactsDomain = b.BaremetalConfig.PrivateServerEksDDomainUrl
+			eksAArtifactsDomain = b.BaremetalConfig.PrivateServerEksADomainUrl
+		case CloudStack:
+			eksDArtifactsDomain = b.CloudstackConfig.PrivateServerEksDDomainUrl
+			eksAArtifactsDomain = b.CloudstackConfig.PrivateServerEksADomainUrl
+		case Nutanix:
+			eksDArtifactsDomain = b.NutanixConfig.PrivateServerEksDDomainUrl
+			eksAArtifactsDomain = b.NutanixConfig.PrivateServerEksADomainUrl
+		}
+		if err = extractAndPrepManifestTarball(b.ManifestTarball, eksDArtifactsDomain, eksAArtifactsDomain); err != nil {
+			log.Fatalf(err.Error())
+		}
+	}
 	buildToolingRepoPath := getBuildToolingPath(cwd)
-	detectedEksaVersion, err := prepBuildToolingRepo(buildToolingRepoPath, b.EKSAReleaseVersion, b.Force)
+	detectedEksaVersion, err := b.prepBuildToolingRepo(buildToolingRepoPath)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
@@ -38,10 +58,14 @@ func (b *BuildOptions) BuildImage() {
 	upstreamImageBuilderProjectPath := filepath.Join(imageBuilderProjectPath, imageBuilderCAPIDirectory)
 	var outputArtifactPath string
 	var outputImageGlob []string
+	eksAReleaseManifestUrl, err := getEksAReleasesManifestURL(b.AirGapped)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
 	commandEnvVars := []string{
 		fmt.Sprintf("%s=%s", releaseBranchEnvVar, b.ReleaseChannel),
 		fmt.Sprintf("%s=%s", eksAReleaseVersionEnvVar, detectedEksaVersion),
-		fmt.Sprintf("%s=%s", eksAReleaseManifestURLEnvVar, getEksAReleasesManifestURL()),
+		fmt.Sprintf("%s=%s", eksAReleaseManifestURLEnvVar, eksAReleaseManifestUrl),
 	}
 
 	log.Printf("Initiating Image Build\n Image OS: %s\n Image OS Version: %s\n Hypervisor: %s\n Firmware: %s\n", b.Os, b.OsVersion, b.Hypervisor, b.Firmware)
@@ -89,6 +113,14 @@ func (b *BuildOptions) BuildImage() {
 	}
 	var outputImageGlobPattern string
 	if b.Hypervisor == VSphere {
+		if b.AirGapped {
+			airGapEnvVars, err := getAirGapCmdEnvVars(b.VsphereConfig.ImageBuilderRepoUrl, detectedEksaVersion, b.ReleaseChannel)
+			if err != nil {
+				log.Fatalf("Error getting air gapped env variables: %v", err)
+			}
+			commandEnvVars = append(commandEnvVars, airGapEnvVars...)
+		}
+
 		// Set proxy on RHSM if available
 		if b.Os == RedHat && b.VsphereConfig.HttpProxy != "" {
 			if err := setRhsmProxy(&b.VsphereConfig.ProxyConfig, &b.VsphereConfig.RhsmConfig); err != nil {
@@ -134,6 +166,14 @@ func (b *BuildOptions) BuildImage() {
 
 		log.Printf("Image Build Successful\n Please find the output artifact at %s\n", outputArtifactPath)
 	} else if b.Hypervisor == Baremetal {
+		if b.AirGapped {
+			airGapEnvVars, err := getAirGapCmdEnvVars(b.BaremetalConfig.ImageBuilderRepoUrl, detectedEksaVersion, b.ReleaseChannel)
+			if err != nil {
+				log.Fatalf("Error getting air gapped env variables: %v", err)
+			}
+			commandEnvVars = append(commandEnvVars, airGapEnvVars...)
+		}
+
 		// Set proxy on RHSM if available
 		if b.Os == RedHat && b.BaremetalConfig.HttpProxy != "" {
 			if err := setRhsmProxy(&b.BaremetalConfig.ProxyConfig, &b.BaremetalConfig.RhsmConfig); err != nil {
@@ -187,10 +227,12 @@ func (b *BuildOptions) BuildImage() {
 			}
 		}
 
-		// Patch firmware config for tool
-		upstreamPatchCommand := fmt.Sprintf("make -C %s patch-repo", imageBuilderProjectPath)
-		if err := executeMakeBuildCommand(upstreamPatchCommand, commandEnvVars...); err != nil {
-			log.Fatalf("Error executing upstream patch command: %v", err)
+		if b.AirGapped {
+			airGapEnvVars, err := getAirGapCmdEnvVars(b.NutanixConfig.ImageBuilderRepoUrl, detectedEksaVersion, b.ReleaseChannel)
+			if err != nil {
+				log.Fatalf("Error getting air gapped env variables: %v", err)
+			}
+			commandEnvVars = append(commandEnvVars, airGapEnvVars...)
 		}
 
 		// Read and set the nutanix connection data
@@ -227,6 +269,13 @@ func (b *BuildOptions) BuildImage() {
 
 		log.Printf("Image Build Successful\n Please find the image uploaded under Nutanix Image Service with name %s\n", b.NutanixConfig.ImageName)
 	} else if b.Hypervisor == CloudStack {
+		if b.AirGapped {
+			airGapEnvVars, err := getAirGapCmdEnvVars(b.CloudstackConfig.ImageBuilderRepoUrl, detectedEksaVersion, b.ReleaseChannel)
+			if err != nil {
+				log.Fatalf("Error getting air gapped env variables: %v", err)
+			}
+			commandEnvVars = append(commandEnvVars, airGapEnvVars...)
+		}
 		// Set proxy on RHSM if available
 		if b.Os == RedHat && b.CloudstackConfig.HttpProxy != "" {
 			if err := setRhsmProxy(&b.CloudstackConfig.ProxyConfig, &b.CloudstackConfig.RhsmConfig); err != nil {

projects/aws/image-builder/builder/constants.go

@@ -29,10 +29,12 @@ const (
 	devEksaReleaseManifestURL        string = "https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/eks-a-release.yaml"
 	devBranchEksaReleaseManifestURL  string = "https://dev-release-assets.eks-anywhere.model-rocket.aws.dev/%s/eks-a-release.yaml"
 	eksDistroProdDomain              string = "distro.eks.amazonaws.com"
+	eksAnywhereAssetsProdDomain      string = "anywhere-assets.eks.amazonaws.com"
 	eksDistroManifestFileNameFormat  string = "eks-d-%s.yaml"
 	eksAnywhereManifestFileName      string = "eks-a-manifest.yaml"
 	eksAnywhereBundlesFileNameFormat string = "eks-a-bundles-%s.yaml"
 	manifestsTarballName             string = "eks-a-manifests.tar"
+	manifestsDirName                 string = "eks-a-d-manifests"
 
 	// Environment variables
 	branchNameEnvVar                      string = "BRANCH_NAME"
@@ -41,6 +43,8 @@ const (
 	releaseBranchEnvVar                   string = "RELEASE_BRANCH"
 	eksAReleaseVersionEnvVar              string = "EKSA_RELEASE_VERSION"
 	eksAReleaseManifestURLEnvVar          string = "EKSA_RELEASE_MANIFEST_URL"
+	eksABundlesURLEnvVar                  string = "EKSA_BUNDLE_MANIFEST_URL"
+	eksDManifestURLEnvVar                 string = "EKSD_MANIFEST_URL"
 	packerAdditionalFilesConfigFileEnvVar string = "PACKER_ADDITIONAL_FILES_VAR_FILES"
 	rhelUsernameEnvVar                    string = "RHSM_USERNAME"
 	rhelPasswordEnvVar                    string = "RHSM_PASSWORD"
@@ -50,6 +54,7 @@ const (
 	packerTypeVarFilesEnvVar              string = "PACKER_TYPE_VAR_FILES"
 	packerNutanixVarFilesEnvVar           string = "PACKER_NUTANIX_VAR_FILES"
 	eksaUseDevReleaseEnvVar               string = "EKSA_USE_DEV_RELEASE"
+	cloneUrlEnvVar                        string = "CLONE_URL"
 
 	// Miscellaneous
 	mainBranch            string = "main"

projects/aws/image-builder/builder/manifests.go

@@ -1,10 +1,13 @@
 package builder
 
 import (
+	"archive/tar"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	releasev1 "github.com/aws/eks-anywhere/release/api/v1alpha1"
 	k8syaml "sigs.k8s.io/yaml"
@@ -20,7 +23,7 @@ func (b *BuildOptions) DownloadManifests() error {
 	}
 
 	buildToolingRepoPath := getBuildToolingPath(cwd)
-	_, err = prepBuildToolingRepo(buildToolingRepoPath, "", b.Force)
+	_, err = b.prepBuildToolingRepo(buildToolingRepoPath)
 	if err != nil {
 		return err
 	}
@@ -58,6 +61,11 @@ func downloadEKSDManifests(outputPath string) error {
 		return err
 	}
 
+	// Create output directory path
+	if err = os.MkdirAll(outputPath, 0755); err != nil {
+		return err
+	}
+
 	for branch, number := range eksDReleaseBranchesWithNumber {
 		manifestUrl := fmt.Sprintf("https://%s/kubernetes-%s/kubernetes-%s-eks-%s.yaml", eksDistroProdDomain, branch, branch, number)
 		manifestFileName := fmt.Sprintf(eksDistroManifestFileNameFormat, branch)
@@ -73,7 +81,16 @@ func downloadEKSDManifests(outputPath string) error {
 
 func downloadEKSAManifests(outputPath string) error {
 	// Download Release manifest
-	releaseManifestUrl := getEksAReleasesManifestURL()
+	releaseManifestUrl, err := getEksAReleasesManifestURL(false)
+	if err != nil {
+		return err
+	}
+
+	// Create output directory path
+	if err = os.MkdirAll(outputPath, 0755); err != nil {
+		return err
+	}
+
 	releaseManifestPath := filepath.Join(outputPath, eksAnywhereManifestFileName)
 	log.Printf("Downloading eks-a release manifest")
 	if err := downloadFile(releaseManifestPath, releaseManifestUrl); err != nil {
@@ -100,3 +117,108 @@ func downloadEKSAManifests(outputPath string) error {
 	}
 	return nil
 }
+
+// extractTarball extracts the provided tarball onto the path
+func extractTarball(tarball, path string) error {
+	tarFile, err := os.Open(tarball)
+	if err != nil {
+		return err
+	}
+	defer tarFile.Close()
+
+	// Create a new directory to extract the tarball into.
+	if err = os.RemoveAll(path); err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(path, 0755)
+	if err != nil {
+		return err
+	}
+
+	// Create a tar reader for the tarball.
+	tarReader := tar.NewReader(tarFile)
+
+	// Iterate over the tarball's entries.
+	for {
+		// Get the next header from the tarball.
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			// We've reached the end of the tarball.
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		// Create a new file for the tar entry.
+		outFile, err := os.Create(filepath.Join(path, header.Name))
+		if err != nil {
+			return err
+		}
+		defer outFile.Close()
+
+		// Copy the tar entry's contents to the new file.
+		_, err = io.Copy(outFile, tarReader)
+		if err != nil {
+			return err
+		}
+
+		outFile.Chmod(os.FileMode(header.Mode))
+	}
+
+	return nil
+}
+
+func extractAndPrepManifestTarball(tarballFile, privateEksDServerDomain, privateEksAServerDomain string) error {
+	log.Println("Manifest tarball provided, extracting to directory")
+	manifestDir, err := getManifestRoot()
+	if err != nil {
+		return fmt.Errorf("Error retrieving manifest root")
+	}
+	if err = extractTarball(tarballFile, manifestDir); err != nil {
+		return fmt.Errorf("Error extracting tarball: %v", err)
+	}
+
+	// Replacing eks-a bundles manifest hostname
+	// These endpoints are used for artifacts like containerd, crictl & etcdadm
+	// Find all eks-a bundles manifest and replace hostname
+	files, err := os.ReadDir(manifestDir)
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		if !file.IsDir() {
+			// Find EKS-D Manifests and replace prod distro domain with private server's
+			absFilePath := filepath.Join(manifestDir, file.Name())
+			eksDManifestFilePattern := strings.ReplaceAll(eksDistroManifestFileNameFormat, "%s", "*")
+			eksDMatch, err := filepath.Match(eksDManifestFilePattern, file.Name())
+			if err != nil {
+				return err
+			}
+
+			if eksDMatch {
+				log.Printf("Replacing EKS-D domain name in file: %s\n", file.Name())
+				if err = replaceStringInFile(absFilePath, fmt.Sprintf("https://%s", eksDistroProdDomain), privateEksDServerDomain); err != nil {
+					return err
+				}
+			}
+
+			// Find EKS-A Bundles Manifests and replace prod anywhere domain with private server's
+			eksABundlesManifestFilePattern := strings.ReplaceAll(eksAnywhereBundlesFileNameFormat, "%s", "*")
+			eksAMatch, err := filepath.Match(eksABundlesManifestFilePattern, file.Name())
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("File: %s, Match: %s, Pattern: %s\n", file.Name(), eksAMatch, eksABundlesManifestFilePattern)
+			if eksAMatch {
+				log.Printf("Replacing EKS-A domain name in file: %s\n", file.Name())
+				if err = replaceStringInFile(absFilePath, fmt.Sprintf("https://%s", eksAnywhereAssetsProdDomain), privateEksAServerDomain); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}