Add build information to docker images and final container and fix im…

…age deployment (#42)

Exaslct now adds build information to the docker images and the final container to easier trace down which image introduces a problem. For that, we include the following information into the images and the container:
- Configuration which was used to create the images and the container
- Dockerfiles of the images
- The package lists that each image installs

Additional minor changes:
- Set lower bounds for the version of the python packages in the Pipfile for exaslct
- Fix image deployment: now all images get deployed instead only the top-level images

Pipfile

@@ -2,15 +2,16 @@
 python_version = "3.6"
 
 [packages]
-luigi = "*"
-jinja2 = "*"
-humanfriendly = "*"
-jsonpickle = "*"
-simplejson = "*"
-docker = "==3.7.2"
-netaddr = "*"
-click = "*"
-requests = "*"
-networkx = "*"
-pydot = "*"
-"stopwatch.py" = "*"
+luigi = ">=2.8.4"
+jinja2 = ">=2.10.1"
+humanfriendly = ">=4.18"
+jsonpickle = ">=1.1"
+simplejson = ">=3.16.0"
+docker = ">=4.0.0"
+netaddr = ">=0.7.19"
+click = ">=7.0"
+requests = ">=2.21.0"
+networkx = ">=2.3"
+pydot = ">=1.4.0"
+"stopwatch.py" = ">=1.0.0"
+gitpython= ">=2.1.0"

exaslct_src/cli/commands/build.py

@@ -37,6 +37,7 @@ def build(flavor_path: Tuple[str, ...],
           temporary_base_directory: str,
           log_build_context_content: bool,
           cache_directory: str,
+          build_name:str,
           source_docker_repository_name: str,
           source_docker_tag_prefix: str,
           source_docker_username: str,
@@ -59,7 +60,8 @@ def build(flavor_path: Tuple[str, ...],
                      log_build_context_content,
                      output_directory,
                      temporary_base_directory,
-                     cache_directory)
+                     cache_directory,
+                     build_name)
     set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
                                  source_docker_tag_prefix, "source")
     set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,

exaslct_src/cli/commands/export.py

@@ -3,7 +3,8 @@
 from click._unicodefun import click
 
 from exaslct_src.cli.cli import cli
-from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, import_build_steps
+from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, \
+    import_build_steps
 from exaslct_src.cli.options \
     import build_options, flavor_options, system_options, release_options, \
     docker_repository_options
@@ -29,6 +30,7 @@ def export(flavor_path: Tuple[str, ...],
            temporary_base_directory: str,
            log_build_context_content: bool,
            cache_directory: str,
+           build_name: str,
            source_docker_repository_name: str,
            source_docker_tag_prefix: str,
            source_docker_username: str,
@@ -51,7 +53,8 @@ def export(flavor_path: Tuple[str, ...],
                      log_build_context_content,
                      output_directory,
                      temporary_base_directory,
-                     cache_directory)
+                     cache_directory,
+                     build_name)
     set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
                                  source_docker_tag_prefix, "source")
     set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,

exaslct_src/cli/commands/push.py

@@ -4,7 +4,8 @@
 
 from exaslct_src.lib.docker_push import DockerPush
 from exaslct_src.cli.cli import cli
-from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, import_build_steps
+from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, \
+    import_build_steps
 from exaslct_src.cli.options \
     import build_options, flavor_options, system_options, docker_repository_options, goal_options
 
@@ -30,6 +31,7 @@ def push(flavor_path: Tuple[str, ...],
          temporary_base_directory: str,
          log_build_context_content: bool,
          cache_directory: str,
+         build_name: str,
          source_docker_repository_name: str,
          source_docker_tag_prefix: str,
          source_docker_username: str,
@@ -51,7 +53,8 @@ def push(flavor_path: Tuple[str, ...],
                      log_build_context_content,
                      output_directory,
                      temporary_base_directory,
-                     cache_directory)
+                     cache_directory,
+                     build_name)
     set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
                                  source_docker_tag_prefix, "source")
     set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,

exaslct_src/cli/commands/run_db_tests.py

@@ -6,7 +6,8 @@
 
 from exaslct_src import TestContainer
 from exaslct_src.cli.cli import cli
-from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, import_build_steps
+from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, \
+    import_build_steps
 from exaslct_src.cli.options \
     import build_options, flavor_options, system_options, release_options, \
     docker_repository_options
@@ -76,6 +77,7 @@ def run_db_test(flavor_path: Tuple[str, ...],
                 temporary_base_directory: str,
                 log_build_context_content: bool,
                 cache_directory: str,
+                build_name: str,
                 source_docker_repository_name: str,
                 source_docker_tag_prefix: str,
                 source_docker_username: str,
@@ -100,7 +102,8 @@ def run_db_test(flavor_path: Tuple[str, ...],
                      log_build_context_content,
                      output_directory,
                      temporary_base_directory,
-                     cache_directory)
+                     cache_directory,
+                     build_name)
     set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
                                  source_docker_tag_prefix, "source")
     set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,

exaslct_src/cli/commands/save.py

@@ -3,7 +3,8 @@
 from click._unicodefun import click
 
 from exaslct_src.cli.cli import cli
-from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, import_build_steps
+from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, \
+    import_build_steps
 from exaslct_src.cli.options \
     import build_options, flavor_options, system_options, goal_options, \
     docker_repository_options
@@ -35,6 +36,7 @@ def save(save_directory: str,
          temporary_base_directory: str,
          log_build_context_content: bool,
          cache_directory: str,
+         build_name: str,
          source_docker_repository_name: str,
          source_docker_tag_prefix: str,
          source_docker_username: str,
@@ -56,7 +58,8 @@ def save(save_directory: str,
                      log_build_context_content,
                      output_directory,
                      temporary_base_directory,
-                     cache_directory)
+                     cache_directory,
+                     build_name)
     set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
                                  source_docker_tag_prefix, "source")
     set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,

exaslct_src/cli/commands/spawn_test_environment.py

@@ -32,6 +32,7 @@ def spawn_test_environment(
                      False,
                      output_directory,
                      temporary_base_directory,
+                     None,
                      None)
     tasks = lambda: [SpawnTestDockerEnvironment(environment_name=environment_name,
                                                 database_port_forward=str(database_port_forward),

exaslct_src/cli/commands/upload.py

@@ -6,7 +6,8 @@
 
 from exaslct_src.lib.upload_container import UploadContainer
 from exaslct_src.cli.cli import cli
-from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, import_build_steps
+from exaslct_src.cli.common import set_build_config, set_docker_repository_config, run_tasks, add_options, \
+    import_build_steps
 from exaslct_src.cli.options \
     import build_options, flavor_options, system_options, release_options, \
     docker_repository_options
@@ -46,6 +47,7 @@ def upload(flavor_path: Tuple[str, ...],
            temporary_base_directory: str,
            log_build_context_content: bool,
            cache_directory: str,
+           build_name: str,
            source_docker_repository_name: str,
            source_docker_tag_prefix: str,
            source_docker_username: str,
@@ -68,7 +70,8 @@ def upload(flavor_path: Tuple[str, ...],
                      log_build_context_content,
                      output_directory,
                      temporary_base_directory,
-                     cache_directory)
+                     cache_directory,
+                     build_name)
     set_docker_repository_config(source_docker_password, source_docker_repository_name, source_docker_username,
                                  source_docker_tag_prefix, "source")
     set_docker_repository_config(target_docker_password, target_docker_repository_name, target_docker_username,

exaslct_src/cli/common.py

@@ -20,7 +20,8 @@ def set_build_config(force_rebuild: bool,
                      log_build_context_content: bool,
                      output_directory: str,
                      temporary_base_directory: str,
-                     cache_directory: str):
+                     cache_directory: str,
+                     build_name: str, ):
     luigi.configuration.get_config().set('build_config', 'force_rebuild', str(force_rebuild))
     luigi.configuration.get_config().set('build_config', 'force_rebuild_from', json.dumps(force_rebuild_from))
     luigi.configuration.get_config().set('build_config', 'force_pull', str(force_pull))
@@ -29,6 +30,8 @@ def set_build_config(force_rebuild: bool,
         luigi.configuration.get_config().set('build_config', 'temporary_base_directory', temporary_base_directory)
     if cache_directory is not None:
         luigi.configuration.get_config().set('build_config', 'cache_directory', cache_directory)
+    if build_name is not None:
+        luigi.configuration.get_config().set('build_config', 'build_name', build_name)
     luigi.configuration.get_config().set('build_config', 'log_build_context_content', str(log_build_context_content))
 
 
@@ -37,7 +40,8 @@ def set_output_directory(output_directory):
         luigi.configuration.get_config().set('build_config', 'output_directory', output_directory)
 
 
-def set_docker_repository_config(docker_password: str, docker_repository_name: str, docker_username: str, tag_prefix: str,
+def set_docker_repository_config(docker_password: str, docker_repository_name: str, docker_username: str,
+                                 tag_prefix: str,
                                  kind: str):
     config_class = f'{kind}_docker_repository_config'
     luigi.configuration.get_config().set(config_class, 'tag_prefix', tag_prefix)

exaslct_src/cli/options.py

@@ -101,6 +101,8 @@
                  help="For Debugging: Logs the files and directories in the build context of a stage"),
     click.option('--cache-directory', default=None, type=click.Path(file_okay=False, dir_okay=True, exists=False),
                  help="Directory from where saved docker images can be loaded"),
+    click.option('--build-name', default=None, type=str,
+                 help="Name of the build. For example: Repository + CI Build Number"),
 ]
 
 system_options = [

exaslct_src/lib/build_config.py

@@ -10,4 +10,5 @@ class build_config(luigi.Config):
     #keep_build_context = luigi.BoolParameter(False)
     temporary_base_directory = luigi.OptionalParameter(None)
     output_directory = luigi.Parameter(".build_output")
-    cache_directory = luigi.OptionalParameter("")
+    cache_directory = luigi.OptionalParameter("")
+    build_name = luigi.OptionalParameter("")

exaslct_src/lib/data/image_info.py

@@ -1,8 +1,10 @@
+from datetime import datetime
 from enum import Enum
 from typing import Dict, Any
 
 from exaslct_src.lib.data.info import Info
 
+
 class ImageState(Enum):
     NOT_EXISTING = 0,
     # After analyze phase or if build phase did touch the image
@@ -18,6 +20,7 @@ class ImageState(Enum):
     WAS_LOADED = 9
     WAS_TAGED = 10
 
+
 class ImageDescription:
     def __init__(self,
                  dockerfile: str,
@@ -31,21 +34,27 @@ def __init__(self,
 
 
 class ImageInfo(Info):
-    DOCKER_TAG_LENGTH_LIMIT=128
+    DOCKER_TAG_LENGTH_LIMIT = 128
     MAX_TAG_SURPLUS = 30
 
     def __init__(self,
                  source_repository_name: str, target_repository_name: str,
-                 source_tag: str, target_tag: str, hash: str,
+                 source_tag: str, target_tag: str,
+                 hash: str, commit: str,
                  image_description: ImageDescription,
+                 build_name:str= "",
+                 build_date_time: datetime=datetime.utcnow(),
                  image_state: ImageState = ImageState.NOT_EXISTING,
                  depends_on_images: Dict[str, "ImageInfo"] = None):
+        self.build_name = build_name
+        self.date_time = str(build_date_time)
+        self.commit = commit
         self.target_repository_name = target_repository_name
         self.source_repository_name = source_repository_name
         self.image_description = image_description
         if isinstance(image_state, ImageState):
             self.image_state = image_state.name
-        elif isinstance(image_state,str):
+        elif isinstance(image_state, str):
             self.image_state = ImageState[image_state].name
         elif image_state is None:
             self.image_state = None
@@ -62,7 +71,7 @@ def check_complete_tag_length(self, tag):
         complete_tag_length_limit = self.DOCKER_TAG_LENGTH_LIMIT + self.MAX_TAG_SURPLUS
         complete_tag = self._create_complete_tag(tag)
         if len(complete_tag) > complete_tag_length_limit:
-            raise Exception(f"Complete Tag to long by {len(complete_tag)-complete_tag_length_limit}:  {complete_tag}")
+            raise Exception(f"Complete Tag to long by {len(complete_tag) - complete_tag_length_limit}:  {complete_tag}")
 
     def get_target_complete_name(self):
         return f"{self.target_repository_name}:{self.get_target_complete_tag()}"
@@ -76,7 +85,7 @@ def get_source_complete_tag(self):
     def get_target_complete_tag(self):
         return self._create_truncated_complete_tag(self.target_tag)
 
-    def _create_truncated_complete_tag(self, tag:str)->str:
+    def _create_truncated_complete_tag(self, tag: str) -> str:
         # we must truncate the tag to 128 characters, because this is the limit of docker tags
         # refer here https://docs.docker.com/engine/reference/commandline/tag/
         complete_tag = self._create_complete_tag(tag)

exaslct_src/lib/docker/build_context_creator.py

@@ -1,12 +1,13 @@
 import os
 import pathlib
 import shutil
+import textwrap
 from pathlib import Path
 
 from jinja2 import Template
 
 from exaslct_src.lib.build_config import build_config
-from exaslct_src.lib.data.image_info import ImageInfo
+from exaslct_src.lib.data.image_info import ImageInfo, ImageState
 
 
 class BuildContextCreator:
@@ -23,8 +24,14 @@ def __init__(self,
     def prepare_build_context_to_temp_dir(self):
         self._copy_build_files_and_directories()
         self._prepare_dockerfile()
+        self._prepare_image_info()
         self._log_build_context()
 
+    def _prepare_image_info(self):
+        self._image_info.image_state = ImageState.WAS_BUILD.name
+        with open(self._temp_directory + "/image_info", "wt") as file:
+            file.write(self._image_info.to_json())
+
     def _prepare_dockerfile(self):
         with open(self._image_description.dockerfile, "rt") as file:
             dockerfile_content = file.read()
@@ -34,6 +41,12 @@ def _prepare_dockerfile(self):
              for key, image_info
              in self._image_info_of_dependencies.items()}
         final_dockerfile = template.render(**image_names_of_dependencies)
+        final_dockerfile += textwrap.dedent(f"""
+        RUN mkdir -p /build_info/image_info
+        COPY image_info /build_info/image_info/{self._image_info.target_tag}
+        RUN mkdir -p /build_info/dockerfiles
+        COPY Dockerfile /build_info/dockerfiles/{self._image_info.target_tag}
+        """)
         with open(self._temp_directory + "/Dockerfile", "wt") as file:
             file.write(final_dockerfile)
 
@@ -59,4 +72,4 @@ def _log_build_context(self):
             shutil.copy(self._temp_directory + "/Dockerfile", str(dockerfile_log_file))
 
     def _get_files_in_build_context(self, temp_directory):
-        return [os.path.join(r, file) for r, d, f in os.walk(temp_directory) for file in f]
+        return [os.path.join(r, file) for r, d, f in os.walk(temp_directory) for file in f]