tcbuilder: add support for bundle property into raw-image

This patch adds support for the bundle property into the raw-image,
this way users can define a docker-compose file to be bundled and
deployed into the new output raw image.

Signed-off-by: Matheus Castello <[email protected]>

tcbuilder/backend/bundle.py

@@ -318,7 +318,7 @@ def get_client(self):
         dind_client = docker.DockerClient(base_url=self.docker_host, tls=tls_config)
         return dind_client
 
-    def save_tar(self, output_file):
+    def save_tar(self, output_file, compress=True):
         """Create compressed tar archive of the Docker images"""
 
         log.info(f"Storing container bundle into \"{self.bundle_dir}\"")
@@ -375,12 +375,16 @@ def save_tar(self, output_file):
             raise OperationFailureError(
                 f"Could not create output tarball in '{output_file_tar}'.")
 
-        log.debug(f"compression_command: {compression_command}")
+        if compress:
+            log.debug(f"compression_command: {compression_command}")
+
+            output_filepath = os.path.join(self.bundle_dir, output_file)
+            if os.path.exists(output_filepath):
+                os.remove(output_filepath)
+            subprocess.run(compression_command, cwd=self.bundle_dir, check=True)
+        else:
+            log.debug(f"Not compressing {output_file_tar}")
 
-        output_filepath = os.path.join(self.bundle_dir, output_file)
-        if os.path.exists(output_filepath):
-            os.remove(output_filepath)
-        subprocess.run(compression_command, cwd=self.bundle_dir, check=True)
 
     def add_cacerts(self, cacerts):
         """Add the required certificate files for a secure registry
@@ -517,7 +521,7 @@ def recursive_yaml_value_check(obj, config_path):
 def download_containers_by_compose_file(
         output_dir, compose_file, host_workdir, output_filename,
         keep_double_dollar_sign=False, platform=None, dind_params=None,
-        use_host_docker=False, show_progress=True):
+        use_host_docker=False, show_progress=True, compress=True):
     """
     Creates a container bundle using Docker (either Host Docker or Docker in Docker)
 
@@ -617,7 +621,7 @@ def download_containers_by_compose_file(
             file.write(yaml.safe_dump(compose_file_data))
 
         log.info("Exporting storage")
-        manager.save_tar(output_filename)
+        manager.save_tar(output_filename, compress)
 
     except docker.errors.APIError as exc:
         raise OperationFailureError(

tcbuilder/backend/deploy.py

@@ -354,6 +354,60 @@ def write_rootfs_to_raw_image(base_raw_img, output_raw_img, base_rootfs_partitio
                 args=(f"{dst_sysroot_dir}/{content}", "/"),
                 loading_msg=f"  Copying /{content}...")
 
+        # handle the combine of docker-storage bundle
+        bundle_dir = "bundle.tmp"
+        if os.path.exists(bundle_dir):
+            bundle_dir = "bundle.tmp"
+            bundle_tar_file_name = "docker-storage.tar"
+            sota_docker_compose_file_name = "docker-compose.yml"
+            sota_target_name_file_name = "target_name"
+            bundle_target_dir = "/ostree/deploy/torizon/var/lib/docker/"
+            sota_docker_compose_target_dir = "/ostree/deploy/torizon/var/sota/storage/docker-compose/"
+            sota_target_name_target_dir = "/ostree/deploy/torizon/var/sota/storage/docker-compose/"
+            sota_docker_compose_file_path = os.path.join(
+                bundle_dir,
+                sota_docker_compose_file_name
+            )
+            bundle_tar_file_path = os.path.join(
+                bundle_dir,
+                bundle_tar_file_name
+            )
+            sota_target_name_file_path = os.path.join(
+                bundle_dir,
+                sota_target_name_file_name
+            )
+
+            if os.path.exists(bundle_tar_file_path):
+                log.info("Copying bundle contents to output image. This may take a few minutes...")
+
+                # create target_name
+                with open(sota_target_name_file_path, 'w') as target_name_fd:
+                    target_name_fd.write("docker-compose.yml")
+
+                if not gfs.is_dir(bundle_target_dir):
+                    gfs.mkdir_p(bundle_target_dir)
+
+                run_with_loading_animation(
+                    func=gfs.tar_in,
+                    args=(bundle_tar_file_path, bundle_target_dir),
+                    loading_msg="  Copying bundle...")
+
+                if not gfs.is_dir(sota_docker_compose_target_dir):
+                    gfs.mkdir_p(sota_docker_compose_target_dir)
+
+                run_with_loading_animation(
+                    func=gfs.copy_in,
+                    args=(sota_docker_compose_file_path, sota_docker_compose_target_dir),
+                    loading_msg="  Copying docker-compose.yml...")
+
+                if not gfs.is_dir(sota_target_name_target_dir):
+                    gfs.mkdir_p(sota_target_name_target_dir)
+
+                run_with_loading_animation(
+                    func=gfs.copy_in,
+                    args=(sota_target_name_file_path, sota_target_name_target_dir),
+                    loading_msg="  Copying target_name...")
+
         gfs.shutdown()
         gfs.close()
     except RuntimeError as gfserr:

tcbuilder/backend/tcbuild.schema.yaml

@@ -280,6 +280,36 @@ properties:
               base-image:
                 type: string
                 description: "base image used for the output raw image"
+              bundle:
+                type: object
+                description: "image bundling configuration for raw image"
+                oneOf:
+                  - properties:
+                      compose-file:
+                        type: string
+                        description: "path of a docker-compose file that will be included in the final image along with the required container images"
+                        tdxMeta: "file:dockercompose"
+                      platform:
+                        type: string
+                        description: "platform for fetching multi-platform container images"
+                      username:
+                        type: string
+                        description: "Docker login username to be used if accessing a private registry is required"
+                      password:
+                        type: string
+                        description: "Docker login password to be used if accessing a private registry is required"
+                      registry:
+                        type: string
+                        description: "Alternative container registry used to access container image"
+                      ca-certificate:
+                        type: string
+                        description: "CA certificate path of alternative container registry"
+                        tdxMeta: "file;cer,crt"
+                      keep-double-dollar-sign:
+                        type: boolean
+                        description: "Keep double dollar sign instead of replacing it with a single one when parsing string values of the compose file"
+                    required: [compose-file]
+                    additionalProperties: false
             required: [local]
             # No extra props inside 'raw-image'
             additionalProperties: false

tcbuilder/cli/build.py

@@ -337,6 +337,14 @@ def handle_raw_image_output(props, storage_dir, union_params, default_base_raw_i
     base_raw_img = props.get("base-image", default_base_raw_image)
     base_rootfs_label = props.get("base-rootfs-label", common.DEFAULT_RAW_ROOTFS_LABEL)
 
+    # Handle the bundle
+    handle_raw_image_bundle_output(
+        "raw-storage",
+        storage_dir,
+        props.get("bundle", {}),
+        props
+    )
+
     deploy_raw_image_params = {
         "ostree_ref": union_params["union_branch"],
         "base_raw_img": base_raw_img,
@@ -458,6 +466,60 @@ def handle_bundle_output(image_dir, storage_dir, bundle_props, tezi_props):
                 shutil.rmtree(bundle_dir)
 
 
+def handle_raw_image_bundle_output(image_dir, storage_dir, bundle_props, raw_props):
+    """Handle the bundle and combine steps of the output generation."""
+
+    if "compose-file" in bundle_props:
+
+        if "platform" in bundle_props:
+            platform = bundle_props["platform"]
+        else:
+            # Detect platform based on OSTree data.
+            platform = common.get_docker_platform(storage_dir)
+
+        # for raw image the bundle.tmp is automatically cleaned
+        bundle_dir = "bundle.tmp"
+        if os.path.exists(bundle_dir):
+            shutil.rmtree(bundle_dir)
+
+        log.info(f"Bundling images to directory {bundle_dir}")
+        try:
+            # Download bundle to temporary directory - currently that directory
+            # must be relative to the work directory.
+            logins = []
+            if bundle_props.get("registry") and bundle_props.get("username"):
+                logins = [(bundle_props.get("registry"),
+                           bundle_props.get("username"),
+                           bundle_props.get("password", ""))]
+            elif bundle_props.get("username"):
+                logins = [(bundle_props.get("username"),
+                           bundle_props.get("password", ""))]
+
+            RegistryOperations.set_logins(logins)
+
+            # CA Certificate of registry
+            if bundle_props.get("registry") and bundle_props.get("ca-certificate"):
+                cacerts = [[bundle_props.get("registry"),
+                            bundle_props.get("ca-certificate")]]
+                RegistryOperations.set_cacerts(cacerts)
+
+            # we download the bundle and leave it there for the next steps
+            download_params = {
+                "output_dir": bundle_dir,
+                "compose_file": bundle_props["compose-file"],
+                "host_workdir": common.get_host_workdir()[0],
+                "use_host_docker": False,
+                "output_filename": common.DOCKER_BUNDLE_FILENAME,
+                "keep_double_dollar_sign": bundle_props.get("keep-double-dollar-sign", False),
+                "platform": platform,
+                "compress": False
+            }
+            download_containers_by_compose_file(**download_params)
+
+        finally:
+            log.debug(f"Completed, bundled to {bundle_dir}")
+
+
 def handle_provisioning(output_dir, prov_props):
     """Handle the provisioning step of the output generation."""