diff --git a/tcbuilder/backend/bundle.py b/tcbuilder/backend/bundle.py index a1155a1..45f61a0 100755 --- a/tcbuilder/backend/bundle.py +++ b/tcbuilder/backend/bundle.py @@ -318,7 +318,7 @@ def get_client(self): dind_client = docker.DockerClient(base_url=self.docker_host, tls=tls_config) return dind_client - def save_tar(self, output_file): + def save_tar(self, output_file, compress=True): """Create compressed tar archive of the Docker images""" log.info(f"Storing container bundle into \"{self.bundle_dir}\"") @@ -375,12 +375,16 @@ def save_tar(self, output_file): raise OperationFailureError( f"Could not create output tarball in '{output_file_tar}'.") - log.debug(f"compression_command: {compression_command}") + if compress: + log.debug(f"compression_command: {compression_command}") + + output_filepath = os.path.join(self.bundle_dir, output_file) + if os.path.exists(output_filepath): + os.remove(output_filepath) + subprocess.run(compression_command, cwd=self.bundle_dir, check=True) + else: + log.debug(f"Not compressing {output_file_tar}") - output_filepath = os.path.join(self.bundle_dir, output_file) - if os.path.exists(output_filepath): - os.remove(output_filepath) - subprocess.run(compression_command, cwd=self.bundle_dir, check=True) def add_cacerts(self, cacerts): """Add the required certificate files for a secure registry @@ -517,7 +521,7 @@ def recursive_yaml_value_check(obj, config_path): def download_containers_by_compose_file( output_dir, compose_file, host_workdir, output_filename, keep_double_dollar_sign=False, platform=None, dind_params=None, - use_host_docker=False, show_progress=True): + use_host_docker=False, show_progress=True, compress=True): """ Creates a container bundle using Docker (either Host Docker or Docker in Docker) @@ -617,7 +621,7 @@ def download_containers_by_compose_file( file.write(yaml.safe_dump(compose_file_data)) log.info("Exporting storage") - manager.save_tar(output_filename) + manager.save_tar(output_filename, compress) except docker.errors.APIError as exc: raise OperationFailureError( diff --git a/tcbuilder/backend/deploy.py b/tcbuilder/backend/deploy.py index daafc36..8fe943f 100644 --- a/tcbuilder/backend/deploy.py +++ b/tcbuilder/backend/deploy.py @@ -354,6 +354,60 @@ def write_rootfs_to_raw_image(base_raw_img, output_raw_img, base_rootfs_partitio args=(f"{dst_sysroot_dir}/{content}", "/"), loading_msg=f" Copying /{content}...") + # handle the combine of docker-storage bundle + bundle_dir = "bundle.tmp" + if os.path.exists(bundle_dir): + bundle_dir = "bundle.tmp" + bundle_tar_file_name = "docker-storage.tar" + sota_docker_compose_file_name = "docker-compose.yml" + sota_target_name_file_name = "target_name" + bundle_target_dir = "/ostree/deploy/torizon/var/lib/docker/" + sota_docker_compose_target_dir = "/ostree/deploy/torizon/var/sota/storage/docker-compose/" + sota_target_name_target_dir = "/ostree/deploy/torizon/var/sota/storage/docker-compose/" + sota_docker_compose_file_path = os.path.join( + bundle_dir, + sota_docker_compose_file_name + ) + bundle_tar_file_path = os.path.join( + bundle_dir, + bundle_tar_file_name + ) + sota_target_name_file_path = os.path.join( + bundle_dir, + sota_target_name_file_name + ) + + if os.path.exists(bundle_tar_file_path): + log.info("Copying bundle contents to output image. This may take a few minutes...") + + # create target_name + with open(sota_target_name_file_path, 'w') as target_name_fd: + target_name_fd.write("docker-compose.yml") + + if not gfs.is_dir(bundle_target_dir): + gfs.mkdir_p(bundle_target_dir) + + run_with_loading_animation( + func=gfs.tar_in, + args=(bundle_tar_file_path, bundle_target_dir), + loading_msg=" Copying bundle...") + + if not gfs.is_dir(sota_docker_compose_target_dir): + gfs.mkdir_p(sota_docker_compose_target_dir) + + run_with_loading_animation( + func=gfs.copy_in, + args=(sota_docker_compose_file_path, sota_docker_compose_target_dir), + loading_msg=" Copying docker-compose.yml...") + + if not gfs.is_dir(sota_target_name_target_dir): + gfs.mkdir_p(sota_target_name_target_dir) + + run_with_loading_animation( + func=gfs.copy_in, + args=(sota_target_name_file_path, sota_target_name_target_dir), + loading_msg=" Copying target_name...") + gfs.shutdown() gfs.close() except RuntimeError as gfserr: diff --git a/tcbuilder/backend/tcbuild.schema.yaml b/tcbuilder/backend/tcbuild.schema.yaml index 18d0bf8..9f4ba41 100644 --- a/tcbuilder/backend/tcbuild.schema.yaml +++ b/tcbuilder/backend/tcbuild.schema.yaml @@ -280,6 +280,36 @@ properties: base-image: type: string description: "base image used for the output raw image" + bundle: + type: object + description: "image bundling configuration for raw image" + oneOf: + - properties: + compose-file: + type: string + description: "path of a docker-compose file that will be included in the final image along with the required container images" + tdxMeta: "file:dockercompose" + platform: + type: string + description: "platform for fetching multi-platform container images" + username: + type: string + description: "Docker login username to be used if accessing a private registry is required" + password: + type: string + description: "Docker login password to be used if accessing a private registry is required" + registry: + type: string + description: "Alternative container registry used to access container image" + ca-certificate: + type: string + description: "CA certificate path of alternative container registry" + tdxMeta: "file;cer,crt" + keep-double-dollar-sign: + type: boolean + description: "Keep double dollar sign instead of replacing it with a single one when parsing string values of the compose file" + required: [compose-file] + additionalProperties: false required: [local] # No extra props inside 'raw-image' additionalProperties: false diff --git a/tcbuilder/cli/build.py b/tcbuilder/cli/build.py index 8a46330..671e47b 100644 --- a/tcbuilder/cli/build.py +++ b/tcbuilder/cli/build.py @@ -337,6 +337,14 @@ def handle_raw_image_output(props, storage_dir, union_params, default_base_raw_i base_raw_img = props.get("base-image", default_base_raw_image) base_rootfs_label = props.get("base-rootfs-label", common.DEFAULT_RAW_ROOTFS_LABEL) + # Handle the bundle + handle_raw_image_bundle_output( + "raw-storage", + storage_dir, + props.get("bundle", {}), + props + ) + deploy_raw_image_params = { "ostree_ref": union_params["union_branch"], "base_raw_img": base_raw_img, @@ -458,6 +466,60 @@ def handle_bundle_output(image_dir, storage_dir, bundle_props, tezi_props): shutil.rmtree(bundle_dir) +def handle_raw_image_bundle_output(image_dir, storage_dir, bundle_props, raw_props): + """Handle the bundle and combine steps of the output generation.""" + + if "compose-file" in bundle_props: + + if "platform" in bundle_props: + platform = bundle_props["platform"] + else: + # Detect platform based on OSTree data. + platform = common.get_docker_platform(storage_dir) + + # for raw image the bundle.tmp is automatically cleaned + bundle_dir = "bundle.tmp" + if os.path.exists(bundle_dir): + shutil.rmtree(bundle_dir) + + log.info(f"Bundling images to directory {bundle_dir}") + try: + # Download bundle to temporary directory - currently that directory + # must be relative to the work directory. + logins = [] + if bundle_props.get("registry") and bundle_props.get("username"): + logins = [(bundle_props.get("registry"), + bundle_props.get("username"), + bundle_props.get("password", ""))] + elif bundle_props.get("username"): + logins = [(bundle_props.get("username"), + bundle_props.get("password", ""))] + + RegistryOperations.set_logins(logins) + + # CA Certificate of registry + if bundle_props.get("registry") and bundle_props.get("ca-certificate"): + cacerts = [[bundle_props.get("registry"), + bundle_props.get("ca-certificate")]] + RegistryOperations.set_cacerts(cacerts) + + # we download the bundle and leave it there for the next steps + download_params = { + "output_dir": bundle_dir, + "compose_file": bundle_props["compose-file"], + "host_workdir": common.get_host_workdir()[0], + "use_host_docker": False, + "output_filename": common.DOCKER_BUNDLE_FILENAME, + "keep_double_dollar_sign": bundle_props.get("keep-double-dollar-sign", False), + "platform": platform, + "compress": False + } + download_containers_by_compose_file(**download_params) + + finally: + log.debug(f"Completed, bundled to {bundle_dir}") + + def handle_provisioning(output_dir, prov_props): """Handle the provisioning step of the output generation."""