Cl 2812 pascom gstreamer build

cerbero/bootstrap/build_tools.py

@@ -100,8 +100,8 @@ def insert_python_site(self):
         except ImportError:
             return
 
-        version = [int(v) for v in stv.__version__.split('.')]
-        if len(version) < 1 or version[:1] < [49]:
+        version = stv.__version__.split('.', 1)
+        if len(version) < 1 or int(version[0]) < 49:
             return
 
         m.warning('detected setuptools >= 49.0.0, installing fallback site.py file. '

cerbero/bootstrap/linux.py

@@ -20,7 +20,7 @@
 from cerbero.bootstrap.bootstrapper import register_system_bootstrapper
 from cerbero.enums import Platform, Architecture, Distro, DistroVersion
 from cerbero.errors import ConfigurationError
-from cerbero.utils import shell
+from cerbero.utils import user_is_root, shell
 from cerbero.utils import messages as m
 
 import shlex
@@ -37,6 +37,8 @@ class UnixBootstrapper (BootstrapperBase):
     def __init__(self, config, offline, assume_yes):
         BootstrapperBase.__init__(self, config, offline)
         self.assume_yes = assume_yes
+        if user_is_root() and 'sudo' in self.tool: # no need for sudo as root user
+          self.tool.remove('sudo')
 
     def start(self, jobs=0):
         for c in self.checks:
@@ -54,7 +56,7 @@ def start(self, jobs=0):
             tool += self.command;
             cmd = tool + self.packages
             m.message("Running command '%s'" % ' '.join(cmd))
-            shell.new_call(cmd)
+            shell.new_call(cmd, interactive=True)
 
 
 class DebianBootstrapper (UnixBootstrapper):
@@ -127,6 +129,8 @@ def __init__(self, config, offline, assume_yes):
                 self.packages.append('glibc.i686')
             if self.config.distro_version in [DistroVersion.FEDORA_24, DistroVersion.FEDORA_25]:
                 self.packages.append('libncurses-compat-libs.i686')
+        if user_is_root():
+            return
         # Use sudo to gain root access on everything except RHEL
         if self.config.distro_version == DistroVersion.REDHAT_6:
             self.tool = ['su', '-c', shlex.join(self.tool)]

cerbero/build/oven.py

@@ -316,8 +316,7 @@ async def build_recipe_steps(step):
         all_steps = ["init"] + [s[1] for s in next(iter(recipes)).steps]
 
         # async queues used for each step
-        loop = asyncio.get_event_loop()
-        default_queue = asyncio.PriorityQueue(loop=loop)
+        default_queue = asyncio.PriorityQueue()
         queues = {step : default_queue for step in all_steps}
 
         # find the install steps for ensuring consistency between all of them
@@ -330,7 +329,7 @@ async def build_recipe_steps(step):
         # allocate jobs
         job_allocation = collections.defaultdict(lambda : 0)
         if self.jobs > 4:
-            queues[BuildSteps.COMPILE[1]] = asyncio.PriorityQueue(loop=loop)
+            queues[BuildSteps.COMPILE[1]] = asyncio.PriorityQueue()
             job_allocation[BuildSteps.COMPILE[1]] = 2
         if self.jobs > 5:
             job_allocation[BuildSteps.COMPILE[1]] = 3
@@ -340,7 +339,7 @@ async def build_recipe_steps(step):
                 # the job allocation since we can run more of them in parallel
                 job_allocation[BuildSteps.COMPILE[1]] = self.jobs // 2
         if self.jobs > 7:
-            install_queue = asyncio.PriorityQueue(loop=loop)
+            install_queue = asyncio.PriorityQueue()
             for step in install_steps:
                 queues[step] = install_queue
             job_allocation[BuildSteps.INSTALL[1]] = 1
@@ -350,13 +349,13 @@ async def build_recipe_steps(step):
             # fast, so we shouldn't parallelize.
             if self.config.platform != Platform.WINDOWS:
                 job_allocation[BuildSteps.EXTRACT[1]] = 1
-                queues[BuildSteps.EXTRACT[1]] = asyncio.PriorityQueue(loop=loop)
+                queues[BuildSteps.EXTRACT[1]] = asyncio.PriorityQueue()
         if self.jobs > 9:
             # Two jobs is the same allocation as fetch-package/bootstrap, which
             # is a good idea to avoid getting bottlenecked if one of the
             # download mirrors is slow.
             job_allocation[BuildSteps.FETCH[1]] = 2
-            queues[BuildSteps.FETCH[1]] = asyncio.PriorityQueue(loop=loop)
+            queues[BuildSteps.FETCH[1]] = asyncio.PriorityQueue()
 
         # async locks used to synchronize step execution
         locks = collections.defaultdict(lambda : None)

cerbero/build/source.py

@@ -21,6 +21,8 @@
 import zipfile
 import tarfile
 import urllib.request, urllib.parse, urllib.error
+import collections
+import asyncio
 from hashlib import sha256
 
 from cerbero.config import Platform, DEFAULT_MIRRORS
@@ -302,6 +304,10 @@ class GitCache (Source):
 
     remotes = None
     commit = None
+    use_submodules = True
+
+    _fetch_locks = collections.defaultdict(asyncio.Lock)
+    _fetch_done = set()
 
     def __init__(self):
         Source.__init__(self)
@@ -322,8 +328,20 @@ def __init__(self):
                                      (self.config.git_root, self.name)
         self.repo_dir = os.path.join(self.config.local_sources, self.name)
         self._previous_env = None
+        # For forced commits in the config
+        self.commit = self.config.recipe_commit(self.name) or self.commit
+        self.remotes.update(self.config.recipes_remotes.get(self.name, {}))
 
     async def fetch(self, checkout=True):
+        # Could have multiple recipes using the same repo.
+        lock = self._fetch_locks[self.repo_dir]
+        async with lock:
+            if self.repo_dir in self._fetch_done:
+                return
+            await self.fetch_impl(checkout)
+            self._fetch_done.add(self.repo_dir)
+
+    async def fetch_impl(self, checkout):
         # First try to get the sources from the cached dir if there is one
         cached_dir = os.path.join(self.config.cached_sources,  self.name)
 
@@ -347,9 +365,9 @@ async def fetch(self, checkout=True):
             if not self.offline:
                 await git.fetch(self.repo_dir, fail=False, logfile=get_logfile(self))
         if checkout:
-            commit = self.config.recipe_commit(self.name) or self.commit
-            await git.checkout(self.repo_dir, commit, logfile=get_logfile(self))
-            await git.submodules_update(self.repo_dir, cached_dir, fail=False, offline=self.offline, logfile=get_logfile(self))
+            await git.checkout(self.repo_dir, self.commit, logfile=get_logfile(self))
+            if self.use_submodules:
+                await git.submodules_update(self.repo_dir, cached_dir, fail=False, offline=self.offline, logfile=get_logfile(self))
 
 
     def built_version(self):
@@ -361,15 +379,25 @@ class Git (GitCache):
     Source handler for git repositories
     '''
 
+    _extract_locks = collections.defaultdict(asyncio.Lock)
+    _extract_done = set()
+
     def __init__(self):
         GitCache.__init__(self)
         if self.commit is None:
             # Used by recipes in recipes/toolchain/
             self.commit = 'origin/sdk-%s' % self.version
-        # For forced commits in the config
-        self.commit = self.config.recipe_commit(self.name) or self.commit
 
     async def extract(self):
+        # Could have multiple recipes using the same repo.
+        lock = self._extract_locks[self.config_src_dir]
+        async with lock:
+            if self.config_src_dir in self._extract_done:
+                return
+            await self.extract_impl()
+            self._extract_done.add(self.config_src_dir)
+
+    async def extract_impl(self):
         if os.path.exists(self.config_src_dir):
             try:
                 commit_hash = git.get_hash(self.repo_dir, self.commit, logfile=get_logfile(self))
@@ -383,7 +411,7 @@ async def extract(self):
             os.makedirs(self.config_src_dir)
 
         # checkout the current version
-        await git.local_checkout(self.config_src_dir, self.repo_dir, self.commit, logfile=get_logfile(self))
+        await git.local_checkout(self.config_src_dir, self.repo_dir, self.commit, logfile=get_logfile(self), use_submodules=self.use_submodules)
 
         for patch in self.patches:
             if not os.path.isabs(patch):

cerbero/enums.py

@@ -20,7 +20,7 @@
 
 
 # Safest place to define this since this file imports very few modules
-CERBERO_VERSION = '1.19.1.1'
+CERBERO_VERSION = '1.19.3'
 
 class Platform:
     ''' Enumeration of supported platforms '''
@@ -142,6 +142,8 @@ class DistroVersion:
     OS_X_MOJAVE = 'osx_mojave'
     OS_X_CATALINA = 'osx_catalina'
     OS_X_BIG_SUR = 'osx_big_sur'
+    OS_X_MONTEREY = 'osx_monterey'
+    # further osx versions are generated automatically
     IOS_8_0 = 'ios_08_0'
     IOS_8_1 = 'ios_08_1'
     IOS_8_2 = 'ios_08_2'

cerbero/ide/vs/env.py

@@ -54,6 +54,16 @@
         ),
         r'VC\Auxiliary\Build\vcvarsall.bat'
     ),
+    'vs17': (
+        (
+            r'Microsoft Visual Studio\2022\Community',
+            r'Microsoft Visual Studio\2022\Professional',
+            r'Microsoft Visual Studio\2022\Enterprise',
+            r'Microsoft Visual Studio\2022\BuildTools',
+            r'Microsoft Visual Studio\2022\Preview',
+        ),
+        r'VC\Auxiliary\Build\vcvarsall.bat'
+    ),
 }
 
 def get_program_files_dir():
@@ -72,6 +82,8 @@ def get_vs_year_version(vcver):
         return '2017'
     if vcver == 'vs16':
         return '2019'
+    if vcver == 'vs17':
+        return '2022'
     raise RuntimeError('Unknown toolset value {!r}'.format(vcver))
 
 def _get_custom_vs_install(vs_version, vs_install_path):

cerbero/utils/__init__.py

@@ -247,7 +247,7 @@ def system_info():
                 distro_version = DistroVersion.UBUNTU_DISCO
             elif distro_version in ['eoan']:
                 distro_version = DistroVersion.UBUNTU_EOAN
-            elif distro_version in ['focal', 'ulyana']:
+            elif distro_version in ['focal'] or (distro_version[0] == 'u' and d[1].startswith('20.')):
                 distro_version = DistroVersion.UBUNTU_FOCAL
             elif d[1].startswith('6.'):
                 distro_version = DistroVersion.DEBIAN_SQUEEZE
@@ -359,7 +359,9 @@ def system_info():
     elif platform == Platform.DARWIN:
         distro = Distro.OS_X
         ver = pplatform.mac_ver()[0]
-        if ver.startswith(('11.', '10.16')):
+        if ver.startswith(('12.')):
+            distro_version = DistroVersion.OS_X_MONTEREY
+        elif ver.startswith(('11.', '10.16')):
             distro_version = DistroVersion.OS_X_BIG_SUR
         elif ver.startswith('10.15'):
             distro_version = DistroVersion.OS_X_CATALINA

cerbero/utils/git.py

@@ -208,7 +208,7 @@ def get_hash(git_dir, commit, logfile=None):
                               fail=False, quiet=True, logfile=logfile).rstrip()
 
 
-async def local_checkout(git_dir, local_git_dir, commit, logfile=None):
+async def local_checkout(git_dir, local_git_dir, commit, logfile=None, use_submodules=True):
     '''
     Clone a repository for a given commit in a different location
 
@@ -224,7 +224,8 @@ async def local_checkout(git_dir, local_git_dir, commit, logfile=None):
     await shell.async_call([GIT, 'clone', local_git_dir, '-s', '-b', branch_name, '.'],
                git_dir, logfile=logfile)
     ensure_user_is_set(git_dir, logfile=logfile)
-    await submodules_update(git_dir, local_git_dir, logfile=logfile)
+    if use_submodules:
+        await submodules_update(git_dir, local_git_dir, logfile=logfile)
 
 def add_remote(git_dir, name, url, logfile=None):
     '''

cerbero/utils/shell.py

@@ -169,7 +169,7 @@ def check_output(cmd, cmd_dir=None, fail=True, logfile=None, env=None, quiet=Fal
     try:
         o = subprocess.check_output(cmd, cwd=cmd_dir, env=env, stderr=stderr)
     except SUBPROCESS_EXCEPTIONS as e:
-        msg = getattr(e, 'output', '')
+        msg = getattr(e, 'output', '').decode(sys.stdout.encoding, errors='replace')
         if not fail:
             return msg
         if logfile:
@@ -181,17 +181,21 @@ def check_output(cmd, cmd_dir=None, fail=True, logfile=None, env=None, quiet=Fal
     return o
 
 
-def new_call(cmd, cmd_dir=None, fail=True, logfile=None, env=None, verbose=False):
+def new_call(cmd, cmd_dir=None, fail=True, logfile=None, env=None, verbose=False, interactive=False):
     cmd = _cmd_string_to_array(cmd, env)
     if logfile:
         logfile.write('Running command {!r}\n'.format(cmd))
         logfile.flush()
     if verbose:
         m.message('Running {!r}\n'.format(cmd))
+    if not interactive:
+        stdin = subprocess.DEVNULL
+    else:
+        stdin = None
     try:
         subprocess.check_call(cmd, cwd=cmd_dir, env=env,
                               stdout=logfile, stderr=subprocess.STDOUT,
-                              stdin=subprocess.DEVNULL)
+                              stdin=stdin)
     except SUBPROCESS_EXCEPTIONS as e:
         returncode = getattr(e, 'returncode', -1)
         if not fail:
@@ -382,6 +386,8 @@ async def download_urllib2(url, destination=None, check_cert=True, overwrite=Fal
     @param destination: destination where the file will be saved
     @type destination: str
     '''
+    # 1MiB chunk size (same as dot:giga for wget)
+    chunk_size = 1024 * 1024
     ctx = None
     if not check_cert:
         import ssl
@@ -392,12 +398,25 @@ async def download_urllib2(url, destination=None, check_cert=True, overwrite=Fal
     if not destination:
         destination = os.path.basename(url)
 
+    from datetime import datetime
+    start_time = datetime.now()
+
     try:
         with open(destination, 'wb') as d:
             req = urllib.request.Request(url)
             req.add_header('User-Agent', USER_AGENT)
-            f = urllib.request.urlopen(req, context=ctx)
-            d.write(f.read())
+            f = urllib.request.urlopen(req, timeout=20, context=ctx)
+            while True:
+                chunk = f.read(chunk_size)
+                if not chunk:
+                    break
+                d.write(chunk)
+                print('.', end='', file=logfile, flush=True)
+            total_time = (datetime.now() - start_time).total_seconds()
+            size = d.tell() / 1024
+            speed = size / total_time
+            print('\nDownloaded {:2,.2f} KiB in {:2,.2f} seconds at {:2,.2f} KiB/s'
+                  .format(size, total_time, speed), file=logfile, flush=True)
     except urllib.error.HTTPError as e:
         if os.path.exists(destination):
             os.remove(destination)