From b93d0c02e83a32707dc20f748918d0d71e78dad5 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Tue, 26 Nov 2024 23:18:08 +0000 Subject: [PATCH] Bug 1932655: upgrade taskgraph to 12.1.0 r=taskgraph-reviewers,mach-reviewers,ahal Mainly to pick up https://github.com/taskcluster/taskgraph/pull/604 and https://github.com/taskcluster/taskgraph/pull/606 Differential Revision: https://phabricator.services.mozilla.com/D229800 UltraBlame original commit: f97cba74e8bd9653347e416ada5e077da97d6c99 --- third_party/python/pyproject.toml | 2 +- third_party/python/requirements.txt | 6 ++--- .../METADATA | 3 +-- .../RECORD | 20 ++++++++--------- .../WHEEL | 2 +- .../entry_points.txt | 0 .../licenses/LICENSE | 0 .../taskgraph/__init__.py | 2 +- .../taskcluster_taskgraph/taskgraph/create.py | 9 +++++++- .../taskgraph/optimize/strategies.py | 3 ++- .../taskgraph/run-task/fetch-content | 22 ++++++++++++++----- .../taskgraph/run-task/run-task | 2 +- third_party/python/uv.lock | 8 +++---- third_party/python/uv.lock.hash | 2 +- 14 files changed, 49 insertions(+), 32 deletions(-) rename third_party/python/taskcluster_taskgraph/{taskcluster_taskgraph-12.0.0.dist-info => taskcluster_taskgraph-12.1.0.dist-info}/METADATA (99%) rename third_party/python/taskcluster_taskgraph/{taskcluster_taskgraph-12.0.0.dist-info => taskcluster_taskgraph-12.1.0.dist-info}/RECORD (88%) rename third_party/python/taskcluster_taskgraph/{taskcluster_taskgraph-12.0.0.dist-info => taskcluster_taskgraph-12.1.0.dist-info}/WHEEL (67%) rename third_party/python/taskcluster_taskgraph/{taskcluster_taskgraph-12.0.0.dist-info => taskcluster_taskgraph-12.1.0.dist-info}/entry_points.txt (100%) rename third_party/python/taskcluster_taskgraph/{taskcluster_taskgraph-12.0.0.dist-info => taskcluster_taskgraph-12.1.0.dist-info}/licenses/LICENSE (100%) diff --git a/third_party/python/pyproject.toml b/third_party/python/pyproject.toml index 7b83dd6c5853..fca143750214 100644 --- a/third_party/python/pyproject.toml +++ b/third_party/python/pyproject.toml @@ -52,7 +52,7 @@ dependencies = [ "setuptools==74.0.0", "six==1.16.0", "slugid==2.0.0", - "taskcluster-taskgraph~=12.0", + "taskcluster-taskgraph~=12.1", "taskcluster-urls==13.0.1", "taskcluster==44.2.2", "toml==0.10.2", diff --git a/third_party/python/requirements.txt b/third_party/python/requirements.txt index 0123d4d57af9..cd95b0ef3e79 100644 --- a/third_party/python/requirements.txt +++ b/third_party/python/requirements.txt @@ -619,9 +619,9 @@ slugid==2.0.0 \ taskcluster==44.2.2 \ --hash=sha256:0266a6a901e1a2ec838984a7f24e7adb6d58f9f2e221a7f613388f8f23f786fc \ --hash=sha256:846d73c597f0f47dd8525c85c8d9bc41111d5200b090690d3f16b2f57c56a2e1 -taskcluster-taskgraph==12.0.0 \ - --hash=sha256:96a6b04bb349621664d15bbb5856018175ce85999d6e0293374603f804c7234b \ - --hash=sha256:7554e8709552918a66f03029134aa99a5c9414a839962d84c00e84ee848ab97e +taskcluster-taskgraph==12.1.0 \ + --hash=sha256:c37b0ff65ab6ae3ae322bf1ecd4d3453db01b37f22164d9a33ab77e634a34d9a \ + --hash=sha256:222ba9f729e6d970de8c177251e3a5f29010332d7cc6ca8967fd8c8b73fa2c1b taskcluster-urls==13.0.1 \ --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \ --hash=sha256:f66dcbd6572a6216ab65949f0fa0b91f2df647918028436c384e6af5cd12ae2b diff --git a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/METADATA b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/METADATA similarity index 99% rename from third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/METADATA rename to third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/METADATA index 183b365d400d..c35abcc9a129 100644 --- a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/METADATA +++ b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/METADATA @@ -1,11 +1,10 @@ Metadata-Version: 2.3 Name: taskcluster-taskgraph -Version: 12.0.0 +Version: 12.1.0 Summary: Build taskcluster taskgraphs Project-URL: Repository, https://github.com/taskcluster/taskgraph Project-URL: Issues, https://github.com/taskcluster/taskgraph/issues Author-email: Mozilla Release Engineering -License-File: LICENSE Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0) diff --git a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/RECORD b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/RECORD similarity index 88% rename from third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/RECORD rename to third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/RECORD index 1e377590741b..aee68afe34e8 100644 --- a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/RECORD +++ b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/RECORD @@ -1,6 +1,6 @@ -taskgraph/__init__.py,sha256=vrsuaZ4DumehJiAAPHD7O481ePEpN5N_9vuv-MEZuM4,718 +taskgraph/__init__.py,sha256=_H9_zoYEWGO9c2L_M63SN76YWczgIqzggz_fETwqurI,718 taskgraph/config.py,sha256=55xqo02311bi9SKpys5izMl8lQaLhRZKtW2AwHyMz40,5172 -taskgraph/create.py,sha256=_zokjSM3ZaO04l2LiMhenE8qXDZVfYvueIIu5hGUhzc,5185 +taskgraph/create.py,sha256=sUKEUAFp3X_1Bs99alh4jw7cFXtDFsbh4T2pAkp5mC0,5315 taskgraph/decision.py,sha256=lAHwcojZUWKIsyW1CgPHYyMtT9ULd0_fQNyUrWimC-w,13897 taskgraph/docker.py,sha256=tP2MRJqAPIUZsCuYAnwg1Qo8AB3N0RjcbYZx07wwMwo,8511 taskgraph/filter_tasks.py,sha256=R7tYXiaVPGIkQ6O1c9-QJrKZ59m9pFXCloUlPraVnZU,866 @@ -25,11 +25,11 @@ taskgraph/loader/default.py,sha256=_bBJG6l04v44Jm5HSIEnVndC05NpNmq5L28QfJHk0wo,1 taskgraph/loader/transform.py,sha256=olUBPjxk3eEIg25sduxlcyqhjoig4ts5kPlT_zs6g9g,2147 taskgraph/optimize/__init__.py,sha256=Oqpq1RW8QzOcu7zaMlNQ3BHT9ws9e_93FWfCqzNcQps,123 taskgraph/optimize/base.py,sha256=ltJBr3WR53DyNnEsvUm9fHjFAGh9gGncmekKD1Dlb-w,20221 -taskgraph/optimize/strategies.py,sha256=nEiUZORwflThbuO8U7CI9IPdmEMXQTUk5var4_T-jco,3576 -taskgraph/run-task/fetch-content,sha256=_1MWiyOChavZHSjYIweh2EGle1WGIlmwZ91GhsW0jRU,33409 +taskgraph/optimize/strategies.py,sha256=tnyDw6-z5S2K7710Vww-EsiqtEqZl5rVYRqg8joJ1dk,3638 +taskgraph/run-task/fetch-content,sha256=_bBs9T6YSNfKL338XqPWmzXbDBc7YzKwG5YMmnnSYvE,33643 taskgraph/run-task/hgrc,sha256=BybWLDR89bWi3pE5T05UqmDHs02CbLypE-omLZWU6Uk,896 taskgraph/run-task/robustcheckout.py,sha256=OhW98v-VKajX55AToL5nHeUANEOPL5fIJgsvsagGfnU,30954 -taskgraph/run-task/run-task,sha256=Cvv3L3LLX_7dL0sBPXayL5fH08Nnz0d-BPUMOv8RVuY,46161 +taskgraph/run-task/run-task,sha256=SS_NegHr87dWNXBx_zK2Tuq2XUMP_t__iUZ5N5rdjB4,46229 taskgraph/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 taskgraph/transforms/base.py,sha256=OYIsK4-pPFPhuK2wV7qyeZSSsPaZdJQuI-ixHf4fbeQ,5155 taskgraph/transforms/cached_tasks.py,sha256=Omg7NGnig-ts3Gq_3oB9YQQ4lQW_KGCP0tL76WBYB3k,2743 @@ -72,8 +72,8 @@ taskgraph/util/vcs.py,sha256=3PZd2fq7YkqP22OnZSpugzIHf6WPALm76CHk-YfK2us,18657 taskgraph/util/verify.py,sha256=RbjKw6aAux5skXf2BihCdH7qyLWAU_plTYQ4GlRLBr0,8983 taskgraph/util/workertypes.py,sha256=dR5NkwvmcY-no_I4l22mw93EKyvGbe_Xv7N9hzkiG1U,2570 taskgraph/util/yaml.py,sha256=29h6RE7JA4z0U2V3WCu-S39lPMBS9CEZEglv1delDvw,1075 -taskcluster_taskgraph-12.0.0.dist-info/METADATA,sha256=Mn2yTn5uqSbHq-sb1kFN0YIMSrUibyixqYr-rMpuKug,4947 -taskcluster_taskgraph-12.0.0.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87 -taskcluster_taskgraph-12.0.0.dist-info/entry_points.txt,sha256=2hxDzE3qq_sHh-J3ROqwpxgQgxO-196phWAQREl2-XA,50 -taskcluster_taskgraph-12.0.0.dist-info/licenses/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725 -taskcluster_taskgraph-12.0.0.dist-info/RECORD,, +taskcluster_taskgraph-12.1.0.dist-info/METADATA,sha256=hHjxzh9-muODo4PH68zt0MYV2KSvF06dBBOa-Gh5t-I,4925 +taskcluster_taskgraph-12.1.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87 +taskcluster_taskgraph-12.1.0.dist-info/entry_points.txt,sha256=2hxDzE3qq_sHh-J3ROqwpxgQgxO-196phWAQREl2-XA,50 +taskcluster_taskgraph-12.1.0.dist-info/licenses/LICENSE,sha256=HyVuytGSiAUQ6ErWBHTqt1iSGHhLmlC8fO7jTCuR8dU,16725 +taskcluster_taskgraph-12.1.0.dist-info/RECORD,, diff --git a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/WHEEL b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/WHEEL similarity index 67% rename from third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/WHEEL rename to third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/WHEEL index cdd68a497cdf..21aaa72961a8 100644 --- a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/WHEEL +++ b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/WHEEL @@ -1,4 +1,4 @@ Wheel-Version: 1.0 -Generator: hatchling 1.25.0 +Generator: hatchling 1.26.3 Root-Is-Purelib: true Tag: py3-none-any diff --git a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/entry_points.txt b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/entry_points.txt similarity index 100% rename from third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/entry_points.txt rename to third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/entry_points.txt diff --git a/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/licenses/LICENSE b/third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/licenses/LICENSE similarity index 100% rename from third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.0.0.dist-info/licenses/LICENSE rename to third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-12.1.0.dist-info/licenses/LICENSE diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/__init__.py b/third_party/python/taskcluster_taskgraph/taskgraph/__init__.py index d1f2da4c1cae..375379507b24 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/__init__.py +++ b/third_party/python/taskcluster_taskgraph/taskgraph/__init__.py @@ -2,7 +2,7 @@ -__version__ = "12.0.0" +__version__ = "12.1.0" diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/create.py b/third_party/python/taskcluster_taskgraph/taskgraph/create.py index d8d33f5268f2..d0505a1c8a9b 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/create.py +++ b/third_party/python/taskcluster_taskgraph/taskgraph/create.py @@ -5,6 +5,7 @@ import json import logging +import os import sys from concurrent import futures @@ -123,7 +124,13 @@ def create_task(session, task_id, label, task_def): return logger.info(f"Creating task with taskId {task_id} for {label}") - res = session.put(f"http://taskcluster/queue/v1/task/{task_id}", json=task_def) + proxy_url = os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip( + "/" + ) + res = session.put( + f"{proxy_url}/queue/v1/task/{task_id}", + json=task_def, + ) if res.status_code != 200: try: logger.error(res.json()["message"]) diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/optimize/strategies.py b/third_party/python/taskcluster_taskgraph/taskgraph/optimize/strategies.py index 7343e8ae597b..42aa488bac56 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/optimize/strategies.py +++ b/third_party/python/taskcluster_taskgraph/taskgraph/optimize/strategies.py @@ -81,7 +81,8 @@ def check(self, files_changed, patterns): def should_remove_task(self, task, params, file_patterns): - if params.get("repository_type") == "hg" and params.get("pushlog_id") == -1: + + if params.get("base_rev") and params.get("head_rev") == params.get("base_rev"): return False changed = self.check(params["files_changed"], file_patterns) diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content b/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content index ae028d767712..adb32096923e 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content +++ b/third_party/python/taskcluster_taskgraph/taskgraph/run-task/fetch-content @@ -209,8 +209,13 @@ def stream_download(url, sha256=None, size=None, headers=None): if size: if size != content_length: raise IntegrityError( - "size mismatch on %s: wanted %d; content-length is %d" - % (url, size, content_length) + "size mismatch on %s: wanted %d; content-length is %d, x-cache-status is: %s" + % ( + url, + size, + content_length, + fh.getheader("x-cache-status"), + ) ) else: size = content_length @@ -435,7 +440,9 @@ def extract_archive(path, dest_dir): log("%s extracted in %.3fs" % (path, time.time() - t0)) -def should_repack_archive(orig: pathlib.Path, dest: pathlib.Path, strip_components=0, add_prefix="") -> bool: +def should_repack_archive( + orig: pathlib.Path, dest: pathlib.Path, strip_components=0, add_prefix="" +) -> bool: """ Determines whether we should attempt to repack an archive based on the naming conventions used in the original file path, the destination file path, and any modifications such as stripping @@ -573,7 +580,9 @@ def repack_archive( ctx.copy_stream(ifh, fh) else: - raise Exception(f"Attempt to repack an archive of unsupported type {orig_typ}") + raise Exception( + f"Attempt to repack an archive of unsupported type {orig_typ}" + ) def fetch_and_extract(url, dest_dir, extract=True, sha256=None, size=None): @@ -660,8 +669,9 @@ def git_checkout_archive( if re.match(r"^[a-fA-F0-9]{40}$", commit): revision = commit else: - ref_output = subprocess.check_output(["git", "ls-remote", repo, - 'refs/heads/' + commit]) + ref_output = subprocess.check_output( + ["git", "ls-remote", repo, "refs/heads/" + commit] + ) revision, _ = ref_output.decode().split(maxsplit=1) log("Fetching revision {}".format(revision)) return _git_checkout_github_archive(dest_path, repo, commit, prefix) diff --git a/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task b/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task index b64aad14c551..3c9a42aa2a70 100644 --- a/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task +++ b/third_party/python/taskcluster_taskgraph/taskgraph/run-task/run-task @@ -39,7 +39,7 @@ from pathlib import Path from threading import Thread from typing import Optional -SECRET_BASEURL_TPL = "http://taskcluster/secrets/v1/secret/{}" +SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip('/')) GITHUB_SSH_FINGERPRINT = ( b"github.com ssh-ed25519 " diff --git a/third_party/python/uv.lock b/third_party/python/uv.lock index 594bdf0e9d6b..3175c44f0a95 100644 --- a/third_party/python/uv.lock +++ b/third_party/python/uv.lock @@ -838,7 +838,7 @@ requires-dist = [ { name = "six", specifier = "==1.16.0" }, { name = "slugid", specifier = "==2.0.0" }, { name = "taskcluster", specifier = "==44.2.2" }, - { name = "taskcluster-taskgraph", specifier = "~=12.0" }, + { name = "taskcluster-taskgraph", specifier = "~=12.1" }, { name = "taskcluster-urls", specifier = "==13.0.1" }, { name = "toml", specifier = "==0.10.2" }, { name = "tomlkit", specifier = "==0.12.3" }, @@ -1329,7 +1329,7 @@ wheels = [ [[package]] name = "taskcluster-taskgraph" -version = "12.0.0" +version = "12.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "appdirs" }, @@ -1343,9 +1343,9 @@ dependencies = [ { name = "taskcluster-urls" }, { name = "voluptuous" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/73/29a26874c065ed2f6e5142cedd6e258efd67729c6c15f7e6f180ae6b8db3/taskcluster_taskgraph-12.0.0.tar.gz", hash = "sha256:96a6b04bb349621664d15bbb5856018175ce85999d6e0293374603f804c7234b", size = 375302 } +sdist = { url = "https://files.pythonhosted.org/packages/2a/f0/d7beb4940c92a6ca0b163e3d5eb5f435586a67e5f6d8924810a06563eb04/taskcluster_taskgraph-12.1.0.tar.gz", hash = "sha256:c37b0ff65ab6ae3ae322bf1ecd4d3453db01b37f22164d9a33ab77e634a34d9a", size = 375550 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/5f/02c48a4890f63def2b746b94a5f0f602478bb84994c3223f6be24357b18f/taskcluster_taskgraph-12.0.0-py3-none-any.whl", hash = "sha256:7554e8709552918a66f03029134aa99a5c9414a839962d84c00e84ee848ab97e", size = 193815 }, + { url = "https://files.pythonhosted.org/packages/73/83/f5ba327be8c2482721029bddf2188450d82d86435a1c5cbc353f2ad5c76b/taskcluster_taskgraph-12.1.0-py3-none-any.whl", hash = "sha256:222ba9f729e6d970de8c177251e3a5f29010332d7cc6ca8967fd8c8b73fa2c1b", size = 193953 }, ] [[package]] diff --git a/third_party/python/uv.lock.hash b/third_party/python/uv.lock.hash index 61996768f203..8969d74db3ef 100644 --- a/third_party/python/uv.lock.hash +++ b/third_party/python/uv.lock.hash @@ -1 +1 @@ -0eecfeb2cd6b7471ac1ad212883553b4b17becf19d6e13b1b73c26d5b064deb2 +a15b1b09555c9bfe6c7a4233dc8b19bf4bf0bfdefde8427c0bbb895558aae6de \ No newline at end of file