Add debugpy support & implement bundle property to raw-image yaml & schema fixes & vs code pylint support

.devcontainer/devcontainer.json

@@ -29,7 +29,8 @@
 				"ms-python.pylint",
 				"eamodio.gitlens",
 				"github.copilot",
-				"jetmartin.bats"
+				"jetmartin.bats",
+				"microhobby.taskrunnercodeplus"
 			]
 		}
 	},

.vscode/launch.json

@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "Python: TorizonCore Builder",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/torizoncore-builder.py",
             "cwd": "/workdir",
@@ -16,6 +16,21 @@
                 "--image-directory",
                 "test"
             ]
+        },
+        {
+            "name": "Python: Attach",
+            "type": "debugpy",
+            "request": "attach",
+            "connect": {
+                "host": "0.0.0.0",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "/builder/"
+                }
+            ]
         }
     ]
 }

.vscode/settings.json

@@ -0,0 +1,13 @@
+{
+    "python.analysis.inlayHints.variableTypes": true,
+    "python.analysis.inlayHints.callArgumentNames": "all",
+    "python.analysis.inlayHints.functionReturnTypes": true,
+    "pylint.enabled": true,
+    "pylint.importStrategy": "fromEnvironment",
+    "pylint.args": [
+        "--disable=missing-module-docstring",
+        "--disable=missing-class-docstring",
+        "--disable=missing-function-docstring",
+        "--disable=fixme"
+    ]
+}

.vscode/tasks.json

@@ -21,6 +21,28 @@
                 "/usr/local/bin/pytest",
                 "--log-cli-level=INFO"
             ]
+        },
+        {
+            "label": "pylint",
+            "type": "shell",
+            "presentation": {
+                "echo": true,
+                "reveal": "always",
+                "focus": false,
+                "panel": "shared",
+                "showReuseMessage": true,
+                "clear": false
+            },
+            "command": "pylint",
+            "args": [
+                "--output-format=colorized",
+                "--disable=missing-module-docstring",
+                "--disable=missing-class-docstring",
+                "--disable=missing-function-docstring",
+                "--disable=fixme",
+                "tcbuilder/**/*.py",
+                "*.py"
+            ]
         }
     ]
-}
+}

requirements_debian.txt

@@ -4,3 +4,4 @@ jsonschema>=3.2.0
 python-dateutil==2.8.2
 pyyaml==5.3.1
 requests-oauthlib==1.3.0
+debugpy==1.8.7

tcbuilder/backend/bundle.py

@@ -33,8 +33,10 @@ def get_compression_command(output_file):
 
     Returns:
         (str, str): output_file without compression ending, compression command
+        (str, None): if output_file doesn't end with any of the compression formats below
     """
     command = None
+    output_file_tar = output_file
     if output_file.endswith(".xz"):
         output_file_tar = output_file[:-3]
         command = ["xz", "-3", "-z", output_file_tar]
@@ -50,6 +52,8 @@ def get_compression_command(output_file):
     elif output_file.endswith(".zst"):
         output_file_tar = output_file[:-4]
         command = ["zstd", "--rm", output_file_tar, "-o", output_file]
+    elif not output_file.endswith(".tar"):
+        output_file_tar = f"{output_file}.tar"
 
     return (output_file_tar, command)
 
@@ -100,7 +104,11 @@ def save_tar(self, output_file):
         output_filepath = os.path.join(self.output_dir, output_file)
         if os.path.exists(output_filepath):
             os.remove(output_filepath)
-        subprocess.run(compression_command, cwd=self.output_dir, check=True)
+
+        if compression_command is not None:
+            subprocess.run(compression_command, cwd=self.output_dir, check=True)
+        else:
+            log.debug(f"Not compressing {output_file_tar}")
 
     def add_cacerts(self, cacerts):
         assert cacerts is None, "`cacerts` should be used with DindManager"
@@ -375,12 +383,16 @@ def save_tar(self, output_file):
             raise OperationFailureError(
                 f"Could not create output tarball in '{output_file_tar}'.")
 
-        log.debug(f"compression_command: {compression_command}")
+        if compression_command is not None:
+            log.debug(f"compression_command: {compression_command}")
+
+            output_filepath = os.path.join(self.bundle_dir, output_file)
+            if os.path.exists(output_filepath):
+                os.remove(output_filepath)
+            subprocess.run(compression_command, cwd=self.bundle_dir, check=True)
+        else:
+            log.debug(f"Not compressing {output_file_tar}")
 
-        output_filepath = os.path.join(self.bundle_dir, output_file)
-        if os.path.exists(output_filepath):
-            os.remove(output_filepath)
-        subprocess.run(compression_command, cwd=self.bundle_dir, check=True)
 
     def add_cacerts(self, cacerts):
         """Add the required certificate files for a secure registry

tcbuilder/backend/combine.py

@@ -4,22 +4,23 @@
 import re
 import datetime
 
+import fnmatch
 import guestfs
 
 from tezi.image import ImageConfig
 from tcbuilder.backend.common import \
     (set_output_ownership, check_licence_acceptance,
-     run_with_loading_animation, DOCKER_BUNDLE_FILENAME)
+     run_with_loading_animation, DOCKER_BUNDLE_TARNAME)
 from tcbuilder.errors import InvalidStateError, InvalidDataError, TorizonCoreBuilderError
 
 log = logging.getLogger("torizon." + __name__)
 
 TARGET_NAME_FILENAME = "target_name"
 DOCKER_FILES_TO_ADD = [
     "docker-compose.yml:/ostree/deploy/torizon/var/sota/storage/docker-compose/",
-    DOCKER_BUNDLE_FILENAME + ":/ostree/deploy/torizon/var/lib/docker/:true",
     TARGET_NAME_FILENAME + ":/ostree/deploy/torizon/var/sota/storage/docker-compose/"
 ]
+DOCKER_BUNDLE_DESTINATION = "/ostree/deploy/torizon/var/lib/docker/:true"
 
 TEZI_PROPS = [
     "name",
@@ -42,6 +43,15 @@
 }
 
 
+# Search in bundle_dir if there is a file named DOCKER_BUNDLE_TARNAME*
+# e.g. DOCKER_BUNDLE_TARNAME, DOCKER_BUNDLE_TARNAME.xz, DOCKER_BUNDLE_TARNAME.gz, etc.
+def check_docker_storage_file(bundle_dir):
+    for filename in os.listdir(bundle_dir):
+        if fnmatch.fnmatch(filename, f"{DOCKER_BUNDLE_TARNAME}*"):
+            return filename
+    return None
+
+
 def set_autoreboot(output_dir, include):
     wrapup_sh = os.path.join(os.path.abspath(output_dir), 'wrapup.sh')
 
@@ -90,7 +100,7 @@ def add_files(tezidir, image_json_filename, filelist, tezi_props):
     config_fname = os.path.join(tezidir, image_json_filename)
     config = ImageConfig(config_fname)
 
-    if config.search_filelist(src=DOCKER_BUNDLE_FILENAME):
+    if config.search_filelist(src=DOCKER_BUNDLE_TARNAME):
         raise InvalidDataError(
             "Currently it is not possible to customize the containers of a base "
             "image already containing container images")
@@ -182,8 +192,9 @@ def check_combine_files(bundle_dir):
 
     files_to_add = []
     if bundle_dir is not None:
-        files_to_add = DOCKER_FILES_TO_ADD
+        files_to_add = DOCKER_FILES_TO_ADD.copy()
         target_name_file = os.path.join(bundle_dir, TARGET_NAME_FILENAME)
+
         if not os.path.exists(target_name_file):
             with open(target_name_file, 'w') as target_name_fd:
                 target_name_fd.write("docker-compose.yml")
@@ -196,6 +207,14 @@ def check_combine_files(bundle_dir):
                 log.error(f"Error: {filename} not found in bundle directory.")
                 return None
 
+        docker_storage_filename = check_docker_storage_file(bundle_dir)
+        if docker_storage_filename is not None:
+            files_to_add.append(
+                f"{docker_storage_filename}:{DOCKER_BUNDLE_DESTINATION}")
+        else:
+            log.error(f"Error: {DOCKER_BUNDLE_TARNAME} not found in bundle directory.")
+            return None
+
     return files_to_add
 
 

tcbuilder/backend/common.py

@@ -31,12 +31,13 @@
 
 log = logging.getLogger("torizon." + __name__)
 
-DOCKER_BUNDLE_FILENAME = "docker-storage.tar.xz"
+DOCKER_BUNDLE_TARNAME = "docker-storage.tar"
 
 # Mapping from architecture to a Docker platform.
 ARCH_TO_DOCKER_PLAT = {
     "aarch64": "linux/arm64",
-    "arm": "linux/arm/v7"
+    "arm": "linux/arm/v7",
+    "x86_64": "linux/amd64"
 }
 
 DEFAULT_DOCKER_PLATFORM = "linux/arm/v7"
@@ -437,7 +438,7 @@ def update_dt_git_repo():
     try:
         repo_obj = git.Repo(os.path.abspath("device-trees"))
         sha = repo_obj.head.object.hexsha
-        repo_obj.remotes["origin"].fetch(repo_obj.active_branch)
+        repo_obj.remotes["origin"].fetch()
         repo_obj.remotes["origin"].pull()
         set_output_ownership("device-trees")
         log.info("'device-trees' is already up to date"

tcbuilder/backend/tcbuild.schema.yaml

@@ -164,7 +164,6 @@ properties:
       filesystem:
         type: array
         description: "directory trees to be applied on top of the ones in the input image"
-        tdxMeta: "path"
         items:
           type: string
           description: "path to directory to be applied"
@@ -280,6 +279,43 @@ properties:
               base-image:
                 type: string
                 description: "base image used for the output raw image"
+              bundle:
+                type: object
+                description: "image bundling configuration for raw image"
+                oneOf:
+                  - properties:
+                      compose-file:
+                        type: string
+                        description: "path of a docker-compose file that will be included in the final image along with the required container images"
+                        tdxMeta: "file;yml,yaml"
+                      platform:
+                        type: string
+                        description: "platform for fetching multi-platform container images"
+                      username:
+                        type: string
+                        description: "Docker login username to be used if accessing a private registry is required"
+                      password:
+                        type: string
+                        description: "Docker login password to be used if accessing a private registry is required"
+                      registry:
+                        type: string
+                        description: "Alternative container registry used to access container image"
+                      ca-certificate:
+                        type: string
+                        description: "CA certificate path of alternative container registry"
+                        tdxMeta: "file;cer,crt"
+                      keep-double-dollar-sign:
+                        type: boolean
+                        description: "Keep double dollar sign instead of replacing it with a single one when parsing string values of the compose file"
+                    required: [compose-file]
+                    additionalProperties: false
+                  - properties:
+                      dir:
+                        type: string
+                        description: "path to directory containing the container images bundle to be combined with the installer image"
+                        tdxMeta: "path"
+                    required: [dir]
+                    additionalProperties: false
             required: [local]
             # No extra props inside 'raw-image'
             additionalProperties: false
@@ -350,7 +386,7 @@ properties:
                       compose-file:
                         type: string
                         description: "path of a docker-compose file that will be included in the final image along with the required container images"
-                        tdxMeta: "file:dockercompose"
+                        tdxMeta: "file;yml,yaml"
                       platform:
                         type: string
                         description: "platform for fetching multi-platform container images"