Extend template to install packages from lockfiles

docker_templates/create.py

@@ -21,6 +21,8 @@
 from ros_buildfarm.templates import get_template_path
 from ros_buildfarm.templates import expand_template, get_wrapper_scripts
 
+from docker_templates.packages import indexUrlTemplateLookup
+
 default_template_prefix_path = ros_buildfarm.templates.template_prefix_path
 
 def expand_template_prefix_path(template_packages):
@@ -44,9 +46,25 @@ def create_files(data, verbose=False):
     # generate Dockerfile
     create_dockerfile(template_name, data, dockerfile_dir, verbose)
 
+    create_lockfiles(data)
+
     if 'entrypoint_name' in data:
         create_entrypoint(data)
 
+def create_lockfiles(data):
+    dockerfile_dir = data['dockerfile_dir']
+    for arch_name, arch_data in data['archs'].items():
+        for package_type, package_list in arch_data.items():
+            lockfile_dir = os.path.join(dockerfile_dir, package_type)
+            if not os.path.exists(lockfile_dir):
+                os.makedirs(lockfile_dir)
+            lockfile_path = os.path.join(lockfile_dir, arch_name + '.txt')
+            if package_list:
+                with open(lockfile_path, 'w') as h:
+                    for package in package_list:
+                        line = f"{package['name']}{package['version']}"
+                        h.write(line)
+
 def create_entrypoint(data):
     # find entrypoint path
     entrypoint_name = data['entrypoint_name']

docker_templates/folders.py

@@ -2,20 +2,33 @@
 import shutil
 import string
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+from em import Interpreter
+
 
 class AltTemplate(string.Template):
     delimiter = '@'
     idpattern = r'[a-z][_a-z0-9]*'
 
 
-def interpret_tempate(tempate, data, tempate_class=string.Template):
-    # Read image perams using platform perams
-    with open(tempate, 'r') as f:
-        value = tempate_class(f.read())
+def interpret_tempate(tempate, data):
 
-    value = value.substitute(data)
+    output = StringIO()
+    try:
+        interpreter = Interpreter(output=output)
+        interpreter.file(open(tempate, 'r'), locals=data)
+        output = output.getvalue()
+    except Exception as e:
+        print("Error processing %s" % tempate)
+        raise
+    finally:
+        interpreter.shutdown()
+        interpreter = None
 
-    return value
+    return output
 
 
 def populate_path(data, path):
@@ -37,21 +50,6 @@ def populate_path(data, path):
     shutil.copy(templates['images'], path)
 
 
-def populate_hooks(data, path):
-    if not os.path.exists(path):
-        os.makedirs(path)
-    for hook_name, hook_tempate in data['hook_names'].items():
-        tags = ' '.join([data['release_name'] + '-' + data['tag_name']])
-        hookfile = interpret_tempate(
-            hook_tempate,
-            {'tags': tags, 'release_name': data['release_name']},
-            AltTemplate)
-        hookfile_path = os.path.join(path, hook_name)
-        with open(hookfile_path, 'w') as f:
-            f.write(hookfile)
-        os.chmod(hookfile_path, 0o744)
-
-
 def populate_paths(manifest, args, create_dockerfiles):
     # For each release
     for release_name, release_data in manifest['release_names'].items():
@@ -66,32 +64,10 @@ def populate_paths(manifest, args, create_dockerfiles):
                 os_code_data['release_name'] = release_name
                 os_code_data['os_name'] = os_name
                 os_code_data['os_code_name'] = os_code_name
+                os_code_data['archs'] = os_code_data['archs']
 
                 populate_path(data=os_code_data, path=dockerfolder_dir)
 
                 if args.auto:
                     # Run the dockerfile generation script
                     create_dockerfiles.main(('dir', '-d' + dockerfolder_dir))
-
-    # Hacks to add hook scripts for osrf repos
-    if 'hacks' in manifest:
-        # For each release
-        for release_name, release_data in manifest['hacks'].items():
-            # For each os supported
-            for os_name, os_data in release_data['os_names'].items():
-                # For each os distro supported
-                for os_code_name, os_code_data in os_data['os_code_names'].items():
-                    if os_code_data['tag_names'] is None:
-                        continue
-                    # For each tag supported:
-                    for tag_name, tag_data in os_code_data['tag_names'].items():
-                        hooksfolder_dir = os.path.join(
-                            release_name, os_name,
-                            os_code_name,
-                            tag_name,
-                            'hooks')
-                        tag_data['release_name'] = release_name
-                        tag_data['os_name'] = os_name
-                        tag_data['os_code_name'] = os_code_name
-                        tag_data['tag_name'] = tag_name
-                        populate_hooks(tag_data, hooksfolder_dir)

docker_templates/packages.py

@@ -16,14 +16,22 @@
 import string
 import re
 import urllib.request
+# import json
 
 import rosdistro
 
 from docker_templates.eol_distro import isDistroEOL
 
-# TODO: think of a better version pattern like
-#  r'\d(?!Version\:\s)(.+)(?=(~\w+\n))' but works without a trailing ~
-version_pattern = r'(?<=Version: )\d+\.\d+\.\d+\-\d+'
+_cached_package_indexs = {}
+
+DockerToAptArchLookup = {
+    'amd64': 'amd64',
+    'arm32v7':'armhf',
+    'arm64v8':'arm64',
+    'i386':'i386'
+}
+
+version_pattern = r'(?<=Version: ).*\n'
 
 sha256_pattern = r'(?<=SHA256: )[0-9a-f]{64}'
 
@@ -42,9 +50,9 @@
 }
 
 packageVersionTemplateLookup = {
-    'gazebo_packages':  string.Template('=$package_version*'),
-    'ros_packages':     string.Template('=$package_version*'),
-    'ros2_packages':    string.Template('=$package_version*'),
+    'gazebo_packages':  string.Template('=$package_version'),
+    'ros_packages':     string.Template('=$package_version'),
+    'ros2_packages':    string.Template('=$package_version'),
 }
 
 packageNameTemplateLookup = {
@@ -56,10 +64,15 @@
 def getPackageIndex(data, package_index_url):
     """Get current online package index"""
 
-    # Download package index
-    req = urllib.request.Request(package_index_url)
-    with urllib.request.urlopen(req) as response:
-        package_index = gzip.decompress(response.read()).decode('utf-8')
+    global _cached_package_indexs
+    if package_index_url in _cached_package_indexs:
+        package_index = _cached_package_indexs[package_index_url]
+    else:
+        # Download package index
+        req = urllib.request.Request(package_index_url)
+        with urllib.request.urlopen(req) as response:
+            package_index = gzip.decompress(response.read()).decode('utf-8')
+        _cached_package_indexs[package_index_url] = package_index
 
     return package_index
 
@@ -76,6 +89,8 @@ def getPackageInfo(package_pattern, package_index):
 
     # Parse for package info
     matchs = re.search(package_pattern, package_index)
+    if matchs is None:
+        return None
     package_info = matchs.group(0)
 
     return package_info
@@ -110,6 +125,8 @@ def getPackageVersions(data, package_index, packages, package_type):
         package_pattern = getPackagePattern(data, package_pattern_template, package)
         package_name = package_name_template.substitute(data, package=package)
         package_info = getPackageInfo(package_pattern, package_index)
+        if package_info is None:
+            continue
         package_sha256 = getPackageSHA256(package_info)
 
         if data['version'] != False:
@@ -123,6 +140,10 @@ def getPackageVersions(data, package_index, packages, package_type):
     return package_versions
 
 def expandPackages(data):
+    # print("################################################################")
+    # print(json.dumps(data,sort_keys=True, indent=4))
+    # print("################################################################")
+    data["archs"] = {i: dict() for i in data["archs"]}
     for package_type in indexUrlTemplateLookup:
         if package_type in data:
             # determine if distro is eol and apply the appropriate index URL template
@@ -147,7 +168,9 @@ def expandPackages(data):
                 package_index_url_template = indexUrlTemplateLookup[package_type + '_snapshots']
             else:
                 package_index_url_template = indexUrlTemplateLookup[package_type]
-            package_index_url = package_index_url_template.substitute(data)
-            package_index = getPackageIndex(data, package_index_url)
-            package_versions = getPackageVersions(data, package_index, data[package_type], package_type)
-            data[package_type] = package_versions
+            for arch in data['archs']:
+                data['arch'] = DockerToAptArchLookup[arch]
+                package_index_url = package_index_url_template.substitute(data)
+                package_index = getPackageIndex(data, package_index_url)
+                package_versions = getPackageVersions(data, package_index, data[package_type], package_type)
+                data['archs'][arch][package_type] = package_versions

docker_templates/templates/docker_images/create_ros_core_image.Dockerfile.em

@@ -52,8 +52,9 @@ ENV LC_ALL C.UTF-8
 ENV ROS_DISTRO @rosdistro_name
 
 @(TEMPLATE(
-    'snippet/label_and_install_package_list.Dockerfile.em',
+    'snippet/copy_and_install_package_list.Dockerfile.em',
     group='ros',
+    package_type='ros_packages',
     packages=ros_packages,
 ))@
 @[if 'entrypoint_name' in locals()]@

docker_templates/templates/docker_images/create_ros_image.Dockerfile.em

@@ -36,8 +36,9 @@ RUN rosdep init && \
 @[if 'ros_packages' in locals()]@
 @[  if ros_packages]@
 @(TEMPLATE(
-    'snippet/label_and_install_package_list.Dockerfile.em',
+    'snippet/copy_and_install_package_list.Dockerfile.em',
     group='ros',
+    package_type='ros_packages',
     packages=ros_packages,
 ))@
 @[  end if]@

docker_templates/templates/docker_images_ros2/create_ros_core_image.Dockerfile.em

@@ -65,8 +65,9 @@ RUN pip3 install -U \
 @[end if]@
 @
 @(TEMPLATE(
-    'snippet/label_and_install_package_list.Dockerfile.em',
-    group='ros2',
+    'snippet/copy_and_install_package_list.Dockerfile.em',
+    group='ros',
+    package_type='ros2_packages',
     packages=ros2_packages,
 ))@
 @[if 'entrypoint_name' in locals()]@

docker_templates/templates/docker_images_ros2/create_ros_image.Dockerfile.em

@@ -53,8 +53,9 @@ RUN pip3 install -U \
 @[end if]@
 @[if 'ros2_packages' in locals()]@
 @(TEMPLATE(
-    'snippet/label_and_install_package_list.Dockerfile.em',
-    group='ros2',
+    'snippet/copy_and_install_package_list.Dockerfile.em',
+    group='ros',
+    package_type='ros2_packages',
     packages=ros2_packages,
 ))@
 @[end if]@

docker_templates/templates/docker_images_ros2/ros1_bridge/create_ros_ros1_bridge_image.Dockerfile.em

@@ -47,15 +47,17 @@ ENV ROS2_DISTRO @ros2distro_name
 
 @[if 'ros_packages' in locals()]@
 @(TEMPLATE(
-    'snippet/label_and_install_package_list.Dockerfile.em',
+    'snippet/copy_and_install_package_list.Dockerfile.em',
     group='ros',
+    package_type='ros_packages',
     packages=ros_packages,
 ))@
 @[end if]@
 @[if 'ros2_packages' in locals()]@
 @(TEMPLATE(
-    'snippet/label_and_install_package_list.Dockerfile.em',
-    group='ros2',
+    'snippet/copy_and_install_package_list.Dockerfile.em',
+    group='ros',
+    package_type='ros2_packages',
     packages=ros2_packages,
 ))@
 @[end if]@

docker_templates/templates/snippet/copy_and_install_package_list.Dockerfile.em

@@ -0,0 +1,11 @@
+@[if packages]@
+# copy @group packages
+ARG TARGETARCH
+COPY @(package_type)/$TARGETARCH.txt /etc/apt/@(group)/@(package_type).txt
+
+# install @group packages
+RUN apt-get update \
+    && xargs -a /etc/apt/@(group)/@(package_type).txt \
+        apt-get install -y --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+@[end if]